From c8f4728a9abbfcbf77e17eed4c589b74eb5ba897 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 14 Oct 2026 15:25:18 +0400 Subject: [PATCH 01/12] feat: add the OriginMatch model to store candidate origins Signed-off-by: tdruez --- ...coveredlicense_review_comments_and_more.py | 49 ++++++++++ scanpipe/models.py | 94 +++++++++++++++++++ 2 files changed, 143 insertions(+) create mode 100644 scanpipe/migrations/0083_alter_discoveredlicense_review_comments_and_more.py diff --git a/scanpipe/migrations/0083_alter_discoveredlicense_review_comments_and_more.py b/scanpipe/migrations/0083_alter_discoveredlicense_review_comments_and_more.py new file mode 100644 index 0000000000..4186f36d50 --- /dev/null +++ b/scanpipe/migrations/0083_alter_discoveredlicense_review_comments_and_more.py @@ -0,0 +1,49 @@ +# Generated by Django 6.1.2 on 2026-10-14 11:24 + +import django.core.validators +import django.db.models.deletion +import scanpipe.models +import uuid +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('scanpipe', '0082_project_labels_json'), + ] + + operations = [ + migrations.AlterField( + model_name='discoveredlicense', + name='review_comments', + field=models.JSONField(blank=True, default=list, help_text='A list of review comments for license detection issues which needs review. These descriptive comments are based on ambiguous detection types and could also offer helpful suggestions on how to review/report these detection issues.', verbose_name='Review Comments'), + ), + migrations.AlterField( + model_name='discoveredpackage', + name='datafile_paths', + field=models.JSONField(blank=True, default=list, help_text='A list of Resource paths for package datafiles which were used to assemble this package.'), + ), + migrations.CreateModel( + name='OriginMatch', + fields=[ + ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')), + ('extra_data', models.JSONField(blank=True, default=dict, help_text='Optional mapping of extra data key/values.')), + ('purl', models.CharField(help_text='Package URL (PURL) of the candidate origin package.', max_length=2048)), + ('match_type', models.CharField(choices=[('package', 'Package'), ('file', 'File'), ('directory', 'Directory'), ('approximate', 'Approximate'), ('snippet', 'Snippet')], help_text='How the resource matched the origin package: the whole package archive, an exact file, a directory, an approximate file, or a snippet.', max_length=20)), + ('tool', models.CharField(help_text='Name of the tool that reported this match.', max_length=50)), + ('tool_version', models.CharField(blank=True, help_text='Version of the tool that reported this match.', max_length=100)), + ('origin_path', models.CharField(blank=True, help_text='Path of the matched file in the origin package.', max_length=2000)), + ('line_ranges', models.JSONField(blank=True, default=list, help_text="Matched line ranges, as a list of mappings with the 'lines' of the resource and the corresponding 'origin_lines' of the origin file.")), + ('similarity', models.PositiveSmallIntegerField(blank=True, help_text='Similarity between the resource and the origin file, from 0 to 100. Empty when the tool does not report it.', null=True, validators=[django.core.validators.MaxValueValidator(100)])), + ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)ss', to='scanpipe.project')), + ('resource', models.ForeignKey(editable=False, help_text='The matched file or directory.', on_delete=django.db.models.deletion.CASCADE, related_name='origin_matches', to='scanpipe.codebaseresource')), + ], + options={ + 'ordering': ['resource__path', '-similarity', 'purl'], + 'indexes': [models.Index(fields=['purl'], name='scanpipe_or_purl_ee2cf2_idx'), models.Index(fields=['match_type'], name='scanpipe_or_match_t_b45ad1_idx'), models.Index(fields=['tool'], name='scanpipe_or_tool_3437f2_idx'), models.Index(fields=['similarity'], name='scanpipe_or_similar_f8430f_idx')], + 'constraints': [models.UniqueConstraint(fields=('resource', 'purl', 'match_type', 'tool', 'origin_path'), name='scanpipe_originmatch_unique_origin_match')], + }, + bases=(scanpipe.models.UpdateMixin, models.Model), + ), + ] diff --git a/scanpipe/models.py b/scanpipe/models.py index 47eba38eba..3198c34eb7 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -42,6 +42,7 @@ from django.core.exceptions import ValidationError from django.core.serializers.json import DjangoJSONEncoder from django.core.validators import EMPTY_VALUES +from django.core.validators import MaxValueValidator from django.db import models from django.db import transaction from django.db.models import Case @@ -5047,3 +5048,96 @@ def create_from_data(cls, package_score, check): details=check.details or [], package_score=package_score, ) + + +class OriginMatchQuerySet(ProjectRelatedQuerySet): + def for_tool(self, tool): + return self.filter(tool=tool) + + def exact_matches(self): + return self.filter(similarity=100) + + def partial_matches(self): + return self.filter(similarity__lt=100) + + +class OriginMatch(UUIDPKModel, ProjectRelatedModel, ExtraDataFieldMixin): + """A candidate origin package for a resource, as reported by an origin tool.""" + + class MatchType(models.TextChoices): + PACKAGE = "package" + FILE = "file" + DIRECTORY = "directory" + APPROXIMATE = "approximate" + SNIPPET = "snippet" + + resource = models.ForeignKey( + CodebaseResource, + related_name="origin_matches", + on_delete=models.CASCADE, + editable=False, + help_text=_("The matched file or directory."), + ) + purl = models.CharField( + max_length=2048, + help_text=_("Package URL (PURL) of the candidate origin package."), + ) + match_type = models.CharField( + max_length=20, + choices=MatchType.choices, + help_text=_( + "How the resource matched the origin package: the whole package " + "archive, an exact file, a directory, an approximate file, or a snippet." + ), + ) + tool = models.CharField( + max_length=50, + help_text=_("Name of the tool that reported this match."), + ) + tool_version = models.CharField( + max_length=100, + blank=True, + help_text=_("Version of the tool that reported this match."), + ) + origin_path = models.CharField( + max_length=2000, + blank=True, + help_text=_("Path of the matched file in the origin package."), + ) + line_ranges = models.JSONField( + default=list, + blank=True, + help_text=_( + "Matched line ranges, as a list of mappings with the 'lines' of the " + "resource and the corresponding 'origin_lines' of the origin file." + ), + ) + similarity = models.PositiveSmallIntegerField( + null=True, + blank=True, + validators=[MaxValueValidator(100)], + help_text=_( + "Similarity between the resource and the origin file, from 0 to 100. " + "Empty when the tool does not report it." + ), + ) + + objects = OriginMatchQuerySet.as_manager() + + class Meta: + ordering = ["resource__path", "-similarity", "purl"] + indexes = [ + models.Index(fields=["purl"]), + models.Index(fields=["match_type"]), + models.Index(fields=["tool"]), + models.Index(fields=["similarity"]), + ] + constraints = [ + models.UniqueConstraint( + fields=["resource", "purl", "match_type", "tool", "origin_path"], + name="%(app_label)s_%(class)s_unique_origin_match", + ), + ] + + def __str__(self): + return f"{self.purl} ({self.match_type} match by {self.tool})" From aec9e800a995cba2180c11faad522ad2a3f9a1b5 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 14 Oct 2026 15:31:22 +0400 Subject: [PATCH 02/12] add a read-only admin for OriginMatch Signed-off-by: tdruez --- scanpipe/admin.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/scanpipe/admin.py b/scanpipe/admin.py index 9d4c0ccb83..9d17fdc4db 100644 --- a/scanpipe/admin.py +++ b/scanpipe/admin.py @@ -27,6 +27,7 @@ from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage +from scanpipe.models import OriginMatch from scanpipe.models import Project @@ -165,6 +166,32 @@ class DiscoveredDependencyAdmin(ScanPipeBaseAdmin): ordering = ["project", "dependency_uid"] +class OriginMatchAdmin(ScanPipeBaseAdmin): + list_display = [ + "purl", + "match_type", + "similarity", + "tool", + "resource", + "project", + ] + search_fields = [ + "uuid", + "purl", + "origin_path", + "resource__path", + ] + list_filter = ["project", "match_type", "tool"] + list_select_related = ["resource", "project"] + ordering = ["project", "resource__path", "-similarity", "purl"] + + def has_change_permission(self, request, obj=None): + return False + + def has_delete_permission(self, request, obj=None): + return False + + class ScanCodeIOAdminSite(admin.AdminSite): site_header = "ScanCode.io administration" site_title = "ScanCode.io administration" @@ -175,3 +202,4 @@ class ScanCodeIOAdminSite(admin.AdminSite): admin_site.register(CodebaseResource, CodebaseResourceAdmin) admin_site.register(DiscoveredPackage, DiscoveredPackageAdmin) admin_site.register(DiscoveredDependency, DiscoveredDependencyAdmin) +admin_site.register(OriginMatch, OriginMatchAdmin) From c67a38703ff605d81d96ea5fbb1e98ba4bd06a4e Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 14 Oct 2026 15:36:18 +0400 Subject: [PATCH 03/12] delete OriginMatch records on project deletion and reset Signed-off-by: tdruez --- scanpipe/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/scanpipe/models.py b/scanpipe/models.py index 3198c34eb7..abd41db7a6 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -706,6 +706,7 @@ def delete_related_objects(self, keep_input=False, keep_webhook=False): self.codebaserelations, self.discovereddependencies, self.discoveredlicenses, + self.originmatchs, self.codebaseresources, self.runs, ] From ede229f6c5e87cc8f29f44d0210727b6b2105b87 Mon Sep 17 00:00:00 2001 From: tdruez Date: Wed, 14 Oct 2026 15:36:31 +0400 Subject: [PATCH 04/12] add unit tests Signed-off-by: tdruez --- scanpipe/tests/test_models.py | 64 +++++++++++++++++++++++++++++++++++ 1 file changed, 64 insertions(+) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 9209196322..8d8b6fc456 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -62,6 +62,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredPackage from scanpipe.models import DiscoveredPackageScore +from scanpipe.models import OriginMatch from scanpipe.models import Project from scanpipe.models import ProjectMessage from scanpipe.models import Run @@ -176,6 +177,13 @@ def test_scanpipe_project_model_delete_related_objects(self): resource = CodebaseResource.objects.create(project=self.project1, path="path") package = DiscoveredPackage.objects.create(project=self.project1) resource.discovered_packages.add(package) + OriginMatch.objects.create( + project=self.project1, + resource=resource, + purl="pkg:pypi/six@1.17.0", + match_type=OriginMatch.MatchType.FILE, + tool="matchcode", + ) delete_log = self.project1.delete_related_objects() expected = { @@ -186,6 +194,7 @@ def test_scanpipe_project_model_delete_related_objects(self): "scanpipe.DiscoveredPackage": 1, "scanpipe.DiscoveredPackage_codebase_resources": 1, "scanpipe.InputSource": 0, + "scanpipe.OriginMatch": 1, "scanpipe.ProjectMessage": 0, "scanpipe.Run": 1, "scanpipe.WebhookDelivery": 0, @@ -3064,6 +3073,41 @@ def test_scanpipe_model_codebase_resource_has_compliance_issue(self): missing = make_resource_file(self.project1, compliance_alert=severities.MISSING) self.assertTrue(missing.has_compliance_issue) + def test_scanpipe_origin_match_queryset_methods(self): + resource = make_resource_file(self.project1, path="six.py") + directory = make_resource_directory(self.project1, path="vendor") + exact_match = OriginMatch.objects.create( + project=self.project1, + resource=resource, + purl="pkg:pypi/six@1.17.0", + match_type=OriginMatch.MatchType.FILE, + tool="matchcode", + similarity=100, + ) + partial_match = OriginMatch.objects.create( + project=self.project1, + resource=resource, + purl="pkg:pypi/six@1.16.0", + match_type=OriginMatch.MatchType.SNIPPET, + tool="scanoss", + similarity=87, + ) + unscored_match = OriginMatch.objects.create( + project=self.project1, + resource=directory, + purl="pkg:npm/ms@2.1.3", + match_type=OriginMatch.MatchType.DIRECTORY, + tool="matchcode", + ) + + origin_matches = self.project1.originmatchs.order_by("purl") + self.assertQuerySetEqual( + origin_matches.for_tool("matchcode"), [unscored_match, exact_match] + ) + self.assertQuerySetEqual(origin_matches.for_tool("scanoss"), [partial_match]) + self.assertQuerySetEqual(origin_matches.exact_matches(), [exact_match]) + self.assertQuerySetEqual(origin_matches.partial_matches(), [partial_match]) + class ScanPipeModelsTransactionTest(TransactionTestCase): """ @@ -3412,3 +3456,23 @@ def test_scanpipe_codebase_resource_create_and_add_package_warnings(self): self.assertEqual(bad_data["version"], message.details["version"]) self.assertEqual(resource.path, message.details["resource_path"]) self.assertIn("in save", message.traceback) + + def test_scanpipe_origin_match_model_unique_origin_match(self): + project1 = make_project("Analysis") + resource = make_resource_file(project1, path="six.py") + origin_match_data = { + "project": project1, + "resource": resource, + "purl": "pkg:pypi/six@1.17.0", + "match_type": OriginMatch.MatchType.FILE, + "tool": "matchcode", + "origin_path": "six-1.17.0/six.py", + } + OriginMatch.objects.create(**origin_match_data) + + with self.assertRaises(IntegrityError): + OriginMatch.objects.create(**origin_match_data) + + other_tool_data = {**origin_match_data, "tool": "scanoss"} + OriginMatch.objects.create(**other_tool_data) + self.assertEqual(2, resource.origin_matches.count()) From 447559d5d1b8c2b4512147d92f83701613f09cee Mon Sep 17 00:00:00 2001 From: tdruez Date: Thu, 15 Oct 2026 16:12:08 +0400 Subject: [PATCH 05/12] add the find_origins pipeline with a SCANOSS importer Signed-off-by: tdruez --- pyproject.toml | 1 + scancodeio/settings.py | 5 + scanpipe/pipelines/find_origins.py | 63 ++++++++ scanpipe/pipes/__init__.py | 5 +- scanpipe/pipes/scanoss.py | 229 +++++++++++++++++++++++++++++ 5 files changed, 302 insertions(+), 1 deletion(-) create mode 100644 scanpipe/pipelines/find_origins.py create mode 100644 scanpipe/pipes/scanoss.py diff --git a/pyproject.toml b/pyproject.toml index 2ffb4ae46f..0e3a5c800e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -162,6 +162,7 @@ collect_symbols_pygments = "scanpipe.pipelines.collect_symbols_pygments:CollectS collect_symbols_tree_sitter = "scanpipe.pipelines.collect_symbols_tree_sitter:CollectSymbolsTreeSitter" enrich_with_purldb = "scanpipe.pipelines.enrich_with_purldb:EnrichWithPurlDB" fetch_scores = "scanpipe.pipelines.fetch_scores:FetchScores" +find_origins = "scanpipe.pipelines.find_origins:FindOrigins" find_vulnerabilities = "scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities" inspect_elf_binaries = "scanpipe.pipelines.inspect_elf_binaries:InspectELFBinaries" inspect_packages = "scanpipe.pipelines.inspect_packages:InspectPackages" diff --git a/scancodeio/settings.py b/scancodeio/settings.py index f869892140..c6fbbb03d3 100644 --- a/scancodeio/settings.py +++ b/scancodeio/settings.py @@ -354,6 +354,11 @@ MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="") MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="") +# SCANOSS integration + +SCANOSS_URL = env.str("SCANOSS_URL", default="").rstrip("/") +SCANOSS_API_KEY = env.str("SCANOSS_API_KEY", default="") + # FederatedCode integration FEDERATEDCODE_GIT_ACCOUNT_URL = env.str( diff --git a/scanpipe/pipelines/find_origins.py b/scanpipe/pipelines/find_origins.py new file mode 100644 index 0000000000..21922e1673 --- /dev/null +++ b/scanpipe/pipelines/find_origins.py @@ -0,0 +1,63 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import scanoss + + +class FindOrigins(Pipeline): + """ + Find the origin of the codebase files using origin detection tools. + + The candidate origin packages reported by each tool are stored as origin + matches, to be combined and reviewed. + + Supported tools: + - SCANOSS: the codebase files are fingerprinted locally, then their paths, + checksums, sizes, and fingerprints are sent to the configured SCANOSS + server. The file contents are not sent. + """ + + download_inputs = False + is_addon = True + + @classmethod + def steps(cls): + return (cls.find_scanoss_origin_matches,) + + @classmethod + def get_availability(cls): + if not scanoss.is_configured(): + return "SCANOSS is not configured." + + if not scanoss.get_uvx_location(): + return "The uvx executable required to run SCANOSS is not available." + + def find_scanoss_origin_matches(self): + """Scan the codebase with SCANOSS and store the reported origin matches.""" + results_location = self.project.get_output_file_path("scanoss", "json") + scanoss.scan_codebase(self.project.codebase_path, results_location) + + origin_match_count = scanoss.load_scanoss_results_from_file( + self.project, results_location + ) + self.log(f"{origin_match_count:,d} SCANOSS origin matches processed.") diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 0951bd0000..8e77d92fae 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -577,7 +577,7 @@ def poll_until_success(check, sleep=10, **kwargs): time.sleep(sleep) -def run_command_safely(command_args): +def run_command_safely(command_args, env=None): """ Execute the external commands following security best practices. @@ -591,6 +591,8 @@ def run_command_safely(command_args): - This does not use the Shell (shell=False) to prevent injection vulnerabilities. - The command should be provided as a list of ``command_args`` arguments. - Only full paths to executable commands should be provided to avoid any ambiguity. + - Secrets go in the optional ``env`` mapping, as the ``command_args`` are + included in the error message. WARNING: If you're incorporating user input into the command, make sure to sanitize and validate the input to prevent any malicious commands from @@ -602,6 +604,7 @@ def run_command_safely(command_args): command_args, capture_output=True, text=True, + env=env, ) if completed_process.returncode: diff --git a/scanpipe/pipes/scanoss.py b/scanpipe/pipes/scanoss.py new file mode 100644 index 0000000000..0a18a3008b --- /dev/null +++ b/scanpipe/pipes/scanoss.py @@ -0,0 +1,229 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import json +import os +import shutil +import tempfile +from pathlib import Path + +from django.conf import settings + +from packageurl import PackageURL + +from scanpipe.models import OriginMatch +from scanpipe.pipes import run_command_safely + +TOOL_NAME = "scanoss" + +SCANOSS_URL = getattr(settings, "SCANOSS_URL", "") +SCANOSS_API_KEY = getattr(settings, "SCANOSS_API_KEY", "") + +# The scanoss-py client is installed in a temporary environment for each scan, +# to keep its dependencies isolated from ScanCode.io. +SCANOSS_PY_VERSION = "1.54.2" + +# Engine flag to report all the candidate components instead of the best match. +DISABLE_BEST_MATCH_FLAG = 256 + +MATCH_TYPE_BY_SCANOSS_ID = { + "file": OriginMatch.MatchType.FILE, + "snippet": OriginMatch.MatchType.SNIPPET, +} + + +def is_configured(): + """Return True if a SCANOSS server URL is set.""" + return bool(SCANOSS_URL) + + +def get_uvx_location(): + """Return the full path of the uvx executable used to run scanoss-py, if any.""" + return shutil.which("uvx") + + +def scan_codebase(codebase_path, output_location): + """ + Scan the ``codebase_path`` directory with SCANOSS and write the raw JSON + results to ``output_location``. + The scanoss-py client sends the file paths, checksums, sizes, and content + fingerprints to the SCANOSS server, not the file contents. + """ + # Only pass what scanoss-py needs to run, not the ScanCode.io secrets. + scanoss_env = { + "PATH": os.environ.get("PATH", ""), + "HOME": os.environ.get("HOME", ""), + "SCANOSS_API_KEY": SCANOSS_API_KEY, + } + + with tempfile.TemporaryDirectory() as cache_directory: + command_args = [ + get_uvx_location(), + f"--cache-dir={cache_directory}", + f"--from=scanoss=={SCANOSS_PY_VERSION}", + "scanoss-py", + "scan", + "--quiet", + f"--apiurl={SCANOSS_URL}", + f"--flags={DISABLE_BEST_MATCH_FLAG}", + f"--output={output_location}", + str(codebase_path), + ] + run_command_safely(command_args, env=scanoss_env) + + +def to_line_range(range_string): + """Return a [start, end] list from a SCANOSS "start-end" range string.""" + start_line, end_line = range_string.split("-") + return [int(start_line), int(end_line)] + + +def parse_line_ranges(lines, oss_lines): + """ + Return the line range mappings of a SCANOSS match from its ``lines`` and + ``oss_lines`` strings, such as "32-161,170-180". SCANOSS lists the ranges + of both strings in the same order. A whole file match has no line ranges. + """ + is_whole_file = lines == "all" + if is_whole_file: + return [] + + return [ + { + "lines": to_line_range(local_range), + "origin_lines": to_line_range(origin_range), + } + for local_range, origin_range in zip( + lines.split(","), oss_lines.split(","), strict=True + ) + ] + + +def parse_similarity(matched): + """Return the similarity as an integer from a SCANOSS "87%" matched string.""" + return int(matched.rstrip("%")) + + +def get_versioned_purl(component): + """ + Return the versioned PURL string of a SCANOSS ``component``, or an empty + string when the component has no PURL. + SCANOSS reports versionless PURLs with the version in a separate field. + """ + purls = component.get("purl") + if not purls: + return "" + + package_url = PackageURL.from_string(purls[0]) + package_url_data = package_url.to_dict() + package_url_data["version"] = package_url.version or component.get("version") + return PackageURL(**package_url_data).to_string() + + +def get_components(scanoss_match): + """ + Return the components of a ``scanoss_match``: the single best match + component by default, or all the candidate components when the scan ran + with the best match disabled. + """ + return scanoss_match.get("components") or [scanoss_match] + + +def get_origin_match_extra_data(scanoss_match, component): + """Return the SCANOSS data kept as OriginMatch extra data.""" + licenses = component.get("licenses", []) + return { + "purls": component.get("purl", []), + "release_date": component.get("release_date", ""), + "url": component.get("url", ""), + "licenses": [ + {"name": license_data["name"], "source": license_data.get("source", "")} + for license_data in licenses + ], + "file_hash": scanoss_match.get("file_hash", ""), + "source_hash": scanoss_match.get("source_hash", ""), + "hits": scanoss_match.get("hits"), + } + + +def make_origin_matches(resource, scanoss_matches): + """Yield unsaved OriginMatch instances for a ``resource`` SCANOSS matches.""" + for scanoss_match in scanoss_matches: + match_type = MATCH_TYPE_BY_SCANOSS_ID.get(scanoss_match.get("id")) + if not match_type: + continue + + line_ranges = parse_line_ranges( + scanoss_match["lines"], scanoss_match["oss_lines"] + ) + similarity = parse_similarity(scanoss_match["matched"]) + tool_version = scanoss_match.get("server", {}).get("version", "") + + for component in get_components(scanoss_match): + purl = get_versioned_purl(component) + if not purl: + continue + + yield OriginMatch( + project=resource.project, + resource=resource, + purl=purl, + match_type=match_type, + tool=TOOL_NAME, + tool_version=tool_version, + origin_path=component.get("file", ""), + line_ranges=line_ranges, + similarity=similarity, + extra_data=get_origin_match_extra_data(scanoss_match, component), + ) + + +def load_scanoss_results(project, scanoss_results): + """ + Create OriginMatch records for the ``project`` resources from the + ``scanoss_results`` mapping of resource path to SCANOSS matches. + Return the number of processed origin matches. + """ + resources = project.codebaseresources.filter(path__in=scanoss_results.keys()) + resources_by_path = {resource.path: resource for resource in resources} + + missing_paths = sorted(set(scanoss_results) - set(resources_by_path)) + if missing_paths: + project.add_warning( + description="SCANOSS results for paths missing from the codebase", + model="OriginMatch", + details={"paths": missing_paths}, + ) + + origin_matches = [ + origin_match + for path, resource in resources_by_path.items() + for origin_match in make_origin_matches(resource, scanoss_results[path]) + ] + OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) + return len(origin_matches) + + +def load_scanoss_results_from_file(project, input_location): + """Load the SCANOSS raw JSON results file at ``input_location``.""" + scanoss_results = json.loads(Path(input_location).read_text()) + return load_scanoss_results(project, scanoss_results) From 214a3400b233affba7cdd45925c3ca909a8c265e Mon Sep 17 00:00:00 2001 From: tdruez Date: Mon, 19 Oct 2026 18:46:29 +0400 Subject: [PATCH 06/12] add a MatchCode.io step to the find_origins pipeline #1926 Signed-off-by: tdruez --- scanpipe/pipelines/find_origins.py | 40 +++++- scanpipe/pipes/matchcode.py | 208 ++++++++++++++++++++++++++++- scanpipe/views.py | 17 ++- 3 files changed, 248 insertions(+), 17 deletions(-) diff --git a/scanpipe/pipelines/find_origins.py b/scanpipe/pipelines/find_origins.py index 21922e1673..2b9c54ee9a 100644 --- a/scanpipe/pipelines/find_origins.py +++ b/scanpipe/pipelines/find_origins.py @@ -21,6 +21,7 @@ # Visit https://github.com/aboutcode-org/scancode.io for support and download. from scanpipe.pipelines import Pipeline +from scanpipe.pipes import matchcode from scanpipe.pipes import scanoss @@ -28,10 +29,13 @@ class FindOrigins(Pipeline): """ Find the origin of the codebase files using origin detection tools. - The candidate origin packages reported by each tool are stored as origin - matches, to be combined and reviewed. + The candidate origin packages reported by each configured tool are stored as + origin matches, to be combined and reviewed. A tool that is not configured is + skipped. Supported tools: + - MatchCode.io: the codebase scan, including file and directory + fingerprints, is sent to the configured MatchCode.io server. - SCANOSS: the codebase files are fingerprinted locally, then their paths, checksums, sizes, and fingerprints are sent to the configured SCANOSS server. The file contents are not sent. @@ -42,18 +46,40 @@ class FindOrigins(Pipeline): @classmethod def steps(cls): - return (cls.find_scanoss_origin_matches,) + return ( + cls.find_matchcode_origin_matches, + cls.find_scanoss_origin_matches, + ) @classmethod def get_availability(cls): - if not scanoss.is_configured(): - return "SCANOSS is not configured." + if not matchcode.is_configured() and not scanoss.is_configured(): + return "No origin detection tool is configured: MatchCode.io or SCANOSS." - if not scanoss.get_uvx_location(): - return "The uvx executable required to run SCANOSS is not available." + def find_matchcode_origin_matches(self): + """Match the codebase with MatchCode.io and store the origin matches.""" + if not matchcode.is_configured(): + self.log("MatchCode.io is not configured, skipping.") + return + + match_results = matchcode.match_codebase(self.project) + matchcode.save_match_results(self.project, match_results) + + origin_match_count = matchcode.load_matchcode_results( + self.project, match_results + ) + self.log(f"{origin_match_count:,d} MatchCode.io origin matches processed.") def find_scanoss_origin_matches(self): """Scan the codebase with SCANOSS and store the reported origin matches.""" + if not scanoss.is_configured(): + self.log("SCANOSS is not configured, skipping.") + return + + if not scanoss.get_uvx_location(): + self.log("The uvx executable required by SCANOSS is missing, skipping.") + return + results_location = self.project.get_output_file_path("scanoss", "json") scanoss.scan_codebase(self.project.codebase_path, results_location) diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index 587d2334b6..b1786f1b97 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -20,19 +20,24 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import json import logging from collections import defaultdict +from pathlib import Path from django.conf import settings import requests +from licensedcode.spans import Span from matchcode_toolkit.fingerprinting import compute_codebase_directory_fingerprints from matchcode_toolkit.fingerprinting import get_file_fingerprint_hashes from matchcode_toolkit.fingerprinting import get_line_by_pos from matchcode_toolkit.fingerprinting import get_stemmed_file_fingerprint_hashes from matchcode_toolkit.stemming import TS_LANGUAGE_CONF from scancode import Scanner +from summarycode.classify import LEGAL_STARTS_ENDS +from scanpipe.models import OriginMatch from scanpipe.pipes import codebase from scanpipe.pipes import flag from scanpipe.pipes import poll_until_success @@ -69,6 +74,17 @@ class MatchCodeIOException(Exception): DEFAULT_TIMEOUT = 60 +TOOL_NAME = "matchcode" + +MATCH_TYPE_BY_STATUS = { + flag.MATCHED_TO_PURLDB_PACKAGE: OriginMatch.MatchType.PACKAGE, + flag.MATCHED_TO_PURLDB_RESOURCE: OriginMatch.MatchType.FILE, + flag.MATCHED_TO_PURLDB_DIRECTORY: OriginMatch.MatchType.DIRECTORY, + flag.APPROXIMATE_MATCHED_TO_PURLDB_RESOURCE: OriginMatch.MatchType.APPROXIMATE, +} + +EXACT_MATCH_TYPES = [OriginMatch.MatchType.PACKAGE, OriginMatch.MatchType.FILE] + def is_configured(): """Return True if the required MatchCode.io settings have been set.""" @@ -422,8 +438,194 @@ def create_packages_from_match_results(project, match_results): if match_resource_extra_data: resource = project.codebaseresources.get(path=match_resource["path"]) # compute line_by_pos for displaying matches in CodebaseResource detail view - with open(resource.location) as f: - content = f.read() - line_by_pos = get_line_by_pos(content) + line_by_pos = get_resource_line_by_pos(resource) match_resource_extra_data["line_by_pos"] = line_by_pos resource.update_extra_data(match_resource_extra_data) + + +def match_codebase(project): + """ + Fingerprint the ``project`` codebase, send it to MatchCode.io, wait for the + matching run to complete, and return the match results. + """ + fingerprint_codebase_resources(project) + fingerprint_codebase_directories(project) + match_url, run_url = send_project_json_to_matchcode(project) + poll_run_url_status(run_url) + return get_match_results(match_url) + + +def save_match_results(project, match_results): + """Write the raw ``match_results`` in the ``project`` output/ directory.""" + output_location = project.get_output_file_path("matchcode", "json") + output_location.write_text(json.dumps(match_results, indent=2)) + + +def get_resource_line_by_pos(resource): + """Return the mapping of token positions to line numbers of a ``resource``.""" + with open(resource.location) as resource_file: + return get_line_by_pos(resource_file.read()) + + +def get_snippet_line_ranges(match_detections, line_by_pos): + """ + Return a (start_line, end_line) tuple for each span of the matched token + positions ``match_detections``, using the ``line_by_pos`` mapping. + """ + return [ + (line_by_pos[span.start], line_by_pos[span.end]) + for span in Span(match_detections).subspans() + ] + + +def is_legal_file_path(path): + """Return True if the file name of ``path`` is the one of a legal file.""" + file_path = Path(path.lower()) + return ( + file_path.name.startswith(LEGAL_STARTS_ENDS) + or file_path.name.endswith(LEGAL_STARTS_ENDS) + or file_path.stem.endswith(LEGAL_STARTS_ENDS) + ) + + +def get_package_extra_data(package_data): + """Return the MatchCode package data kept as OriginMatch extra data.""" + return { + "package_uid": package_data.get("package_uid", ""), + "release_date": package_data.get("release_date") or "", + "declared_license_expression": package_data.get( + "declared_license_expression", "" + ), + "download_url": package_data.get("download_url", ""), + } + + +def make_snippet_origin_matches( + resource, matched_snippets, package_data_by_purl, tool_version +): + """ + Yield unsaved OriginMatch instances for the ``resource`` snippet matches. + Snippets matched in legal files are skipped, as a license text is a + license clue, not an origin. + MatchCode snippet similarity is kept as extra data only, as its scale is not + comparable with the other match types. + """ + line_by_pos = get_resource_line_by_pos(resource) + + for matched_snippet in matched_snippets: + origin_path = matched_snippet["resource"] + if is_legal_file_path(origin_path): + continue + + line_ranges = [ + {"lines": [start_line, end_line]} + for start_line, end_line in get_snippet_line_ranges( + matched_snippet["match_detections"], line_by_pos + ) + ] + + purl = matched_snippet["package"] + extra_data = get_package_extra_data(package_data_by_purl.get(purl, {})) + extra_data["snippet_similarity"] = matched_snippet["similarity"] + + yield OriginMatch( + project=resource.project, + resource=resource, + purl=purl, + match_type=OriginMatch.MatchType.SNIPPET, + tool=TOOL_NAME, + tool_version=tool_version, + origin_path=origin_path, + line_ranges=line_ranges, + extra_data=extra_data, + ) + + +def make_package_origin_matches( + resource, match_type, package_uids, package_data_by_uid, tool_version +): + """Yield unsaved OriginMatch instances for the ``resource`` matched packages.""" + similarity = 100 if match_type in EXACT_MATCH_TYPES else None + seen_purls = set() + + for package_uid in package_uids: + package_data = package_data_by_uid[package_uid] + purl = package_data["purl"] + if purl in seen_purls: + continue + seen_purls.add(purl) + + yield OriginMatch( + project=resource.project, + resource=resource, + purl=purl, + match_type=match_type, + tool=TOOL_NAME, + tool_version=tool_version, + similarity=similarity, + extra_data=get_package_extra_data(package_data), + ) + + +def make_origin_matches( + resource, resource_results, package_data_by_uid, package_data_by_purl, tool_version +): + """Yield unsaved OriginMatch instances from a ``resource`` match results.""" + matched_snippets = (resource_results.get("extra_data") or {}).get( + "matched_snippets" + ) + if matched_snippets: + yield from make_snippet_origin_matches( + resource, matched_snippets, package_data_by_purl, tool_version + ) + return + + match_type = MATCH_TYPE_BY_STATUS.get(resource_results.get("status")) + if not match_type: + return + + yield from make_package_origin_matches( + resource, + match_type, + resource_results["for_packages"], + package_data_by_uid, + tool_version, + ) + + +def load_matchcode_results(project, match_results): + """ + Create OriginMatch records for the ``project`` resources from the MatchCode.io + ``match_results``. Return the number of processed origin matches. + """ + packages_data = match_results.get("packages", []) + package_data_by_uid = { + package_data["package_uid"]: package_data for package_data in packages_data + } + package_data_by_purl = { + package_data["purl"]: package_data for package_data in packages_data + } + tool_version = match_results["headers"][0]["tool_version"] + + resources_results = [ + resource_results + for resource_results in match_results.get("files", []) + if resource_results.get("for_packages") + ] + matched_paths = [resource_results["path"] for resource_results in resources_results] + resources = project.codebaseresources.filter(path__in=matched_paths) + resources_by_path = {resource.path: resource for resource in resources} + + origin_matches = [ + origin_match + for resource_results in resources_results + for origin_match in make_origin_matches( + resources_by_path[resource_results["path"]], + resource_results, + package_data_by_uid, + package_data_by_purl, + tool_version, + ) + ] + OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) + return len(origin_matches) diff --git a/scanpipe/views.py b/scanpipe/views.py index 7981f2a7d7..04c5e1483a 100644 --- a/scanpipe/views.py +++ b/scanpipe/views.py @@ -63,7 +63,6 @@ from aboutcode.api_auth.views import BaseRevokeAPIKeyView from django_filters.views import FilterView from django_htmx.http import HttpResponseClientRedirect -from licensedcode.spans import Span from packageurl.contrib.django.models import PACKAGE_URL_FIELDS from scancodeio.auth import ConditionalLoginRequired @@ -103,6 +102,7 @@ from scanpipe.pipes import compliance from scanpipe.pipes import count_group_by from scanpipe.pipes import filename_now +from scanpipe.pipes import matchcode from scanpipe.pipes import output from scanpipe.pipes import purldb from scanpipe.settings import scanpipe_settings @@ -2160,6 +2160,9 @@ def get_matched_snippet_annotations(resource): if not matched_snippets: return [] + # line_by_pos is stored as JSON and keys in JSON are always strings + line_by_pos = {int(position): line for position, line in line_by_pos.items()} + snippet_annotations = [] for snippet in matched_snippets: package = snippet.get("package", "") @@ -2169,14 +2172,14 @@ def get_matched_snippet_annotations(resource): f"package: {package}\nresource: {resource}\nsimilarity: {similarity}\n" ) - # convert qspan from list of ints to Spans - qspan = Span(snippet["match_detections"]) - for span in qspan.subspans(): - # line_by_pos is stored as JSON and keys in JSON are always strings + line_ranges = matchcode.get_snippet_line_ranges( + snippet["match_detections"], line_by_pos + ) + for start_line, end_line in line_ranges: snippet_annotations.append( { - "start_line": line_by_pos[str(span.start)], - "end_line": line_by_pos[str(span.end)], + "start_line": start_line, + "end_line": end_line, "text": text, } ) From 0a03e772dac722b5d8c12be1f55e0fd6f4a9b4e9 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 20 Oct 2026 08:37:42 +0400 Subject: [PATCH 07/12] define exact origin matches by match type and warn on missing uvx #1926 Signed-off-by: tdruez --- scanpipe/models.py | 10 +++++----- scanpipe/pipelines/find_origins.py | 8 ++++++-- scanpipe/pipes/matchcode.py | 4 +--- scanpipe/pipes/scanoss.py | 23 +++++++++++------------ scanpipe/tests/test_models.py | 12 +++++------- 5 files changed, 28 insertions(+), 29 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index abd41db7a6..343a672cab 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -5052,14 +5052,11 @@ def create_from_data(cls, package_score, check): class OriginMatchQuerySet(ProjectRelatedQuerySet): - def for_tool(self, tool): - return self.filter(tool=tool) - def exact_matches(self): - return self.filter(similarity=100) + return self.filter(match_type__in=self.model.EXACT_MATCH_TYPES) def partial_matches(self): - return self.filter(similarity__lt=100) + return self.exclude(match_type__in=self.model.EXACT_MATCH_TYPES) class OriginMatch(UUIDPKModel, ProjectRelatedModel, ExtraDataFieldMixin): @@ -5072,6 +5069,9 @@ class MatchType(models.TextChoices): APPROXIMATE = "approximate" SNIPPET = "snippet" + # A directory match can be approximate, so only these types are exact. + EXACT_MATCH_TYPES = [MatchType.PACKAGE, MatchType.FILE] + resource = models.ForeignKey( CodebaseResource, related_name="origin_matches", diff --git a/scanpipe/pipelines/find_origins.py b/scanpipe/pipelines/find_origins.py index 2b9c54ee9a..a43e25c3f7 100644 --- a/scanpipe/pipelines/find_origins.py +++ b/scanpipe/pipelines/find_origins.py @@ -77,13 +77,17 @@ def find_scanoss_origin_matches(self): return if not scanoss.get_uvx_location(): - self.log("The uvx executable required by SCANOSS is missing, skipping.") + message = "The uvx executable required by SCANOSS is missing, skipping." + self.log(message) + self.project.add_warning( + description=message, model="find_scanoss_origin_matches" + ) return results_location = self.project.get_output_file_path("scanoss", "json") scanoss.scan_codebase(self.project.codebase_path, results_location) - origin_match_count = scanoss.load_scanoss_results_from_file( + origin_match_count = scanoss.load_scanoss_results( self.project, results_location ) self.log(f"{origin_match_count:,d} SCANOSS origin matches processed.") diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index b1786f1b97..573108b19a 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -83,8 +83,6 @@ class MatchCodeIOException(Exception): flag.APPROXIMATE_MATCHED_TO_PURLDB_RESOURCE: OriginMatch.MatchType.APPROXIMATE, } -EXACT_MATCH_TYPES = [OriginMatch.MatchType.PACKAGE, OriginMatch.MatchType.FILE] - def is_configured(): """Return True if the required MatchCode.io settings have been set.""" @@ -545,7 +543,7 @@ def make_package_origin_matches( resource, match_type, package_uids, package_data_by_uid, tool_version ): """Yield unsaved OriginMatch instances for the ``resource`` matched packages.""" - similarity = 100 if match_type in EXACT_MATCH_TYPES else None + similarity = 100 if match_type in OriginMatch.EXACT_MATCH_TYPES else None seen_purls = set() for package_uid in package_uids: diff --git a/scanpipe/pipes/scanoss.py b/scanpipe/pipes/scanoss.py index 0a18a3008b..f00d09091c 100644 --- a/scanpipe/pipes/scanoss.py +++ b/scanpipe/pipes/scanoss.py @@ -126,14 +126,18 @@ def parse_similarity(matched): def get_versioned_purl(component): """ Return the versioned PURL string of a SCANOSS ``component``, or an empty - string when the component has no PURL. + string when the component has no valid PURL. SCANOSS reports versionless PURLs with the version in a separate field. """ purls = component.get("purl") if not purls: return "" - package_url = PackageURL.from_string(purls[0]) + try: + package_url = PackageURL.from_string(purls[0]) + except ValueError: + return "" + package_url_data = package_url.to_dict() package_url_data["version"] = package_url.version or component.get("version") return PackageURL(**package_url_data).to_string() @@ -197,12 +201,13 @@ def make_origin_matches(resource, scanoss_matches): ) -def load_scanoss_results(project, scanoss_results): +def load_scanoss_results(project, results_location): """ - Create OriginMatch records for the ``project`` resources from the - ``scanoss_results`` mapping of resource path to SCANOSS matches. - Return the number of processed origin matches. + Create OriginMatch records for the ``project`` resources from the SCANOSS + raw JSON results file at ``results_location``, a mapping of resource path to + SCANOSS matches. Return the number of processed origin matches. """ + scanoss_results = json.loads(Path(results_location).read_text()) resources = project.codebaseresources.filter(path__in=scanoss_results.keys()) resources_by_path = {resource.path: resource for resource in resources} @@ -221,9 +226,3 @@ def load_scanoss_results(project, scanoss_results): ] OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) return len(origin_matches) - - -def load_scanoss_results_from_file(project, input_location): - """Load the SCANOSS raw JSON results file at ``input_location``.""" - scanoss_results = json.loads(Path(input_location).read_text()) - return load_scanoss_results(project, scanoss_results) diff --git a/scanpipe/tests/test_models.py b/scanpipe/tests/test_models.py index 8d8b6fc456..b83f78a09a 100644 --- a/scanpipe/tests/test_models.py +++ b/scanpipe/tests/test_models.py @@ -3084,15 +3084,15 @@ def test_scanpipe_origin_match_queryset_methods(self): tool="matchcode", similarity=100, ) - partial_match = OriginMatch.objects.create( + snippet_match = OriginMatch.objects.create( project=self.project1, resource=resource, purl="pkg:pypi/six@1.16.0", match_type=OriginMatch.MatchType.SNIPPET, tool="scanoss", - similarity=87, + similarity=100, ) - unscored_match = OriginMatch.objects.create( + directory_match = OriginMatch.objects.create( project=self.project1, resource=directory, purl="pkg:npm/ms@2.1.3", @@ -3101,12 +3101,10 @@ def test_scanpipe_origin_match_queryset_methods(self): ) origin_matches = self.project1.originmatchs.order_by("purl") + self.assertQuerySetEqual(origin_matches.exact_matches(), [exact_match]) self.assertQuerySetEqual( - origin_matches.for_tool("matchcode"), [unscored_match, exact_match] + origin_matches.partial_matches(), [directory_match, snippet_match] ) - self.assertQuerySetEqual(origin_matches.for_tool("scanoss"), [partial_match]) - self.assertQuerySetEqual(origin_matches.exact_matches(), [exact_match]) - self.assertQuerySetEqual(origin_matches.partial_matches(), [partial_match]) class ScanPipeModelsTransactionTest(TransactionTestCase): From fbe153f74e33d06beb200119ac3ecc350ce8a38b Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 20 Oct 2026 08:42:00 +0400 Subject: [PATCH 08/12] compute the MatchCode snippet similarity as the code line coverage #1926 Signed-off-by: tdruez --- scanpipe/pipes/matchcode.py | 27 ++++++++++++++++++++++----- 1 file changed, 22 insertions(+), 5 deletions(-) diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index 573108b19a..f7bef0773c 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -476,6 +476,21 @@ def get_snippet_line_ranges(match_detections, line_by_pos): ] +def get_line_coverage(line_ranges, line_by_pos): + """ + Return the percentage of the code lines covered by the (start_line, end_line) + ``line_ranges``, the code lines being the lines with tokens in ``line_by_pos``. + This matches the SCANOSS similarity: the percentage of the file matched. + """ + code_lines = set(line_by_pos.values()) + covered_lines = { + line + for start_line, end_line in line_ranges + for line in range(start_line, end_line + 1) + } + return round(100 * len(covered_lines & code_lines) / len(code_lines)) + + def is_legal_file_path(path): """Return True if the file name of ``path`` is the one of a legal file.""" file_path = Path(path.lower()) @@ -505,8 +520,8 @@ def make_snippet_origin_matches( Yield unsaved OriginMatch instances for the ``resource`` snippet matches. Snippets matched in legal files are skipped, as a license text is a license clue, not an origin. - MatchCode snippet similarity is kept as extra data only, as its scale is not - comparable with the other match types. + The MatchCode snippet similarity is kept as extra data only, as its scale is + not comparable with the other tools. """ line_by_pos = get_resource_line_by_pos(resource) @@ -515,11 +530,12 @@ def make_snippet_origin_matches( if is_legal_file_path(origin_path): continue + snippet_line_ranges = get_snippet_line_ranges( + matched_snippet["match_detections"], line_by_pos + ) line_ranges = [ {"lines": [start_line, end_line]} - for start_line, end_line in get_snippet_line_ranges( - matched_snippet["match_detections"], line_by_pos - ) + for start_line, end_line in snippet_line_ranges ] purl = matched_snippet["package"] @@ -535,6 +551,7 @@ def make_snippet_origin_matches( tool_version=tool_version, origin_path=origin_path, line_ranges=line_ranges, + similarity=get_line_coverage(snippet_line_ranges, line_by_pos), extra_data=extra_data, ) From 1f2bbef7e1c1c641af7d463aa1197105d313c2f8 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 20 Oct 2026 08:55:24 +0400 Subject: [PATCH 09/12] document the find_origins pipeline and the SCANOSS settings #1926 Signed-off-by: tdruez --- docs/application-settings.rst | 25 +++++++++++++++++++++++++ docs/built-in-pipelines.rst | 14 ++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/docs/application-settings.rst b/docs/application-settings.rst index 094cd4952d..73f7d4f34c 100644 --- a/docs/application-settings.rst +++ b/docs/application-settings.rst @@ -381,6 +381,31 @@ API key using ``MATCHCODEIO_API_KEY``:: MATCHCODEIO_API_KEY=insert_your_api_key_here +.. _scancodeio_settings_scanoss: + +SCANOSS +^^^^^^^ + +`SCANOSS `_ identifies the origin of files and code +snippets. You can use the free public service, a premium service, or your own +instance. + +To configure your local environment, set the ``SCANOSS_URL`` in your ``.env`` +file:: + + SCANOSS_URL=https://api.osskb.org + +If your SCANOSS service requires authentication, you can provide the API key +using ``SCANOSS_API_KEY``:: + + SCANOSS_API_KEY=insert_your_api_key_here + +.. note:: + The SCANOSS client is installed in a temporary environment for each scan, + using the ``uvx`` executable, which requires access to PyPI. + Only the file paths, checksums, sizes, and fingerprints are sent to the + SCANOSS service, not the file contents. + .. _scancodeio_settings_federatedcode: FEDERATEDCODE diff --git a/docs/built-in-pipelines.rst b/docs/built-in-pipelines.rst index 644f81ee03..62f54af556 100644 --- a/docs/built-in-pipelines.rst +++ b/docs/built-in-pipelines.rst @@ -140,6 +140,20 @@ Enrich With PurlDB (addon) :members: :member-order: bysource +.. _pipeline_find_origins: + +Find Origins (addon) +-------------------- +.. warning:: + This pipeline requires access to at least one origin detection tool. + Refer to :ref:`scancodeio_settings_matchcodeio` and + :ref:`scancodeio_settings_scanoss` to configure access to MatchCode.io and + SCANOSS in your ScanCode.io instance. + +.. autoclass:: scanpipe.pipelines.find_origins.FindOrigins() + :members: + :member-order: bysource + .. _pipeline_find_vulnerabilities: Find Vulnerabilities (addon) From b403b8bc4dc44c119b4cd3c025aa159aff7bd886 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 20 Oct 2026 08:55:34 +0400 Subject: [PATCH 10/12] add tests for the find_origins pipeline and its MatchCode and SCANOSS steps #1926 Signed-off-by: tdruez --- .../data/find-origins/matchcode-results.json | 713 ++++++++++++++++++ .../data/find-origins/scanoss-results.json | 105 +++ scanpipe/tests/data/find-origins/scheduler.js | 162 ++++ scanpipe/tests/pipes/test_matchcode.py | 179 +++++ scanpipe/tests/pipes/test_pipes.py | 15 + scanpipe/tests/pipes/test_scanoss.py | 187 +++++ scanpipe/tests/test_pipelines.py | 106 +++ 7 files changed, 1467 insertions(+) create mode 100644 scanpipe/tests/data/find-origins/matchcode-results.json create mode 100644 scanpipe/tests/data/find-origins/scanoss-results.json create mode 100644 scanpipe/tests/data/find-origins/scheduler.js create mode 100644 scanpipe/tests/pipes/test_scanoss.py diff --git a/scanpipe/tests/data/find-origins/matchcode-results.json b/scanpipe/tests/data/find-origins/matchcode-results.json new file mode 100644 index 0000000000..ba73f0a421 --- /dev/null +++ b/scanpipe/tests/data/find-origins/matchcode-results.json @@ -0,0 +1,713 @@ +{ + "headers": [ + { + "tool_name": "scanpipe", + "tool_version": "38.1.0" + } + ], + "packages": [ + { + "purl": "pkg:npm/lodash@4.17.21", + "type": "npm", + "namespace": "", + "name": "lodash", + "version": "4.17.21", + "package_uid": "pkg:npm/lodash@4.17.21?uuid=e3ceae14-876f-4659-8cf3-836122f64823", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz" + }, + { + "purl": "pkg:npm/ms@2.1.3", + "type": "npm", + "namespace": "", + "name": "ms", + "version": "2.1.3", + "package_uid": "pkg:npm/ms@2.1.3?uuid=484380e1-4383-40ac-927a-df8e82bf3ee6", + "release_date": null, + "declared_license_expression": "mit", + "download_url": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + }, + { + "purl": "pkg:npm/ms@2.1.3", + "type": "npm", + "namespace": "", + "name": "ms", + "version": "2.1.3", + "package_uid": "pkg:npm/ms@2.1.3?uuid=59bbd3a1-0d87-4783-9bc3-41c2d399c473", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz" + }, + { + "purl": "pkg:pypi/six@1.17.0", + "type": "pypi", + "namespace": "", + "name": "six", + "version": "1.17.0", + "package_uid": "pkg:pypi/six@1.17.0?uuid=b25b1d26-be0c-49aa-b04d-55f529612878", + "release_date": null, + "declared_license_expression": "", + "download_url": "https://files.pythonhosted.org/packages/source/s/six/six-1.17.0.tar.gz" + } + ], + "files": [ + { + "path": "embedded-package/vendor/ms", + "type": "directory", + "status": "matched-to-purldb-directory", + "for_packages": [ + "pkg:npm/ms@2.1.3?uuid=59bbd3a1-0d87-4783-9bc3-41c2d399c473" + ], + "extra_data": {} + }, + { + "path": "embedded-package/vendor/ms/index.js", + "type": "file", + "status": "matched-to-purldb-directory", + "for_packages": [ + "pkg:npm/ms@2.1.3?uuid=484380e1-4383-40ac-927a-df8e82bf3ee6", + "pkg:npm/ms@2.1.3?uuid=59bbd3a1-0d87-4783-9bc3-41c2d399c473" + ], + "extra_data": {} + }, + { + "path": "exact-copy/myapp/_vendor/six.py", + "type": "file", + "status": "matched-to-purldb-resource", + "for_packages": [ + "pkg:pypi/six@1.17.0?uuid=b25b1d26-be0c-49aa-b04d-55f529612878" + ], + "extra_data": {} + }, + { + "path": "no-origin/src/greenhouse_watering.py", + "type": "file", + "status": "scanned", + "for_packages": [], + "extra_data": {} + }, + { + "path": "snippet/src/scheduler.js", + "type": "file", + "status": "scanned", + "for_packages": [ + "pkg:npm/lodash@4.17.21?uuid=e3ceae14-876f-4659-8cf3-836122f64823" + ], + "extra_data": { + "matched_snippets": [ + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "lodash-4.17.21.tgz-extract/package/debounce.js", + "similarity": 0.004526935264825713, + "match_detections": [ + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356 + ] + }, + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "lodash-4.17.21.tgz-extract/package/lodash.js", + "similarity": 0.004526935264825713, + "match_detections": [ + 59, + 60, + 61, + 62, + 63, + 64, + 65, + 66, + 67, + 68, + 69, + 70, + 71, + 72, + 73, + 74, + 75, + 76, + 77, + 78, + 79, + 80, + 81, + 82, + 83, + 84, + 85, + 86, + 87, + 88, + 89, + 90, + 91, + 92, + 93, + 94, + 95, + 96, + 97, + 98, + 99, + 100, + 101, + 102, + 103, + 104, + 105, + 106, + 107, + 108, + 109, + 110, + 111, + 112, + 113, + 114, + 115, + 116, + 117, + 118, + 119, + 120, + 121, + 122, + 123, + 124, + 125, + 126, + 127, + 128, + 129, + 130, + 131, + 132, + 133, + 134, + 135, + 136, + 137, + 138, + 139, + 140, + 141, + 142, + 143, + 144, + 145, + 146, + 147, + 148, + 149, + 150, + 151, + 152, + 153, + 154, + 155, + 156, + 157, + 158, + 159, + 160, + 161, + 162, + 163, + 164, + 165, + 166, + 167, + 168, + 169, + 170, + 171, + 172, + 173, + 174, + 175, + 176, + 177, + 178, + 179, + 180, + 181, + 182, + 183, + 184, + 185, + 186, + 187, + 188, + 189, + 190, + 191, + 192, + 193, + 194, + 195, + 196, + 197, + 198, + 199, + 200, + 201, + 202, + 203, + 204, + 205, + 206, + 207, + 208, + 209, + 210, + 211, + 212, + 213, + 214, + 215, + 216, + 217, + 218, + 219, + 220, + 221, + 222, + 223, + 224, + 225, + 226, + 227, + 228, + 229, + 230, + 231, + 232, + 233, + 234, + 235, + 236, + 237, + 238, + 239, + 240, + 241, + 242, + 243, + 244, + 245, + 246, + 247, + 248, + 249, + 250, + 251, + 252, + 253, + 254, + 255, + 256, + 257, + 258, + 259, + 260, + 261, + 262, + 263, + 264, + 265, + 266, + 267, + 268, + 269, + 270, + 271, + 272, + 273, + 274, + 275, + 276, + 277, + 278, + 279, + 280, + 281, + 282, + 283, + 284, + 285, + 286, + 287, + 288, + 289, + 290, + 291, + 292, + 293, + 294, + 295, + 296, + 297, + 298, + 299, + 300, + 301, + 302, + 303, + 304, + 305, + 306, + 307, + 308, + 309, + 310, + 311, + 312, + 313, + 314, + 315, + 316, + 317, + 318, + 319, + 320, + 321, + 322, + 323, + 324, + 325, + 326, + 327, + 328, + 329, + 330, + 331, + 332, + 333, + 334, + 335, + 336, + 337, + 338, + 339, + 340, + 341, + 342, + 343, + 344, + 345, + 346, + 347, + 348, + 349, + 350, + 351, + 352, + 353, + 354, + 355, + 356 + ] + } + ] + } + } + ] +} diff --git a/scanpipe/tests/data/find-origins/scanoss-results.json b/scanpipe/tests/data/find-origins/scanoss-results.json new file mode 100644 index 0000000000..ee0a5dcaac --- /dev/null +++ b/scanpipe/tests/data/find-origins/scanoss-results.json @@ -0,0 +1,105 @@ +{ + "exact-copy/myapp/_vendor/six.py": [ + { + "component": "six", + "file": "six.py", + "file_hash": "71e8581c332473a17735159487901fb5", + "file_url": " ", + "id": "file", + "latest": "39e2879", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/MIT.txt", + "copyleft": "no", + "name": "MIT", + "osadl_updated": "2026-02-12T16:31:00+0000", + "patent_hints": "no", + "source": "component_declared", + "url": "https://spdx.org/licenses/MIT.html" + } + ], + "lines": "all", + "matched": "100%", + "oss_lines": "all", + "purl": [ + "pkg:github/benjaminp/six", + "pkg:pypi/six", + "pkg:golang/github.com/benjaminp/six", + "pkg:conda/six" + ], + "release_date": "2024-12-03", + "server": { + "kb_version": { + "daily": "26.10.19", + "monthly": "26.09" + }, + "version": "5.4.27" + }, + "source_hash": "71e8581c332473a17735159487901fb5", + "status": "pending", + "url": "https://github.com/benjaminp/six", + "url_hash": "a67779a691efb149b7ee5c7d9dd4f93e", + "vendor": "benjaminp", + "version": "1.17.0" + } + ], + "modified-copy/src/inih/ini.c": [ + { + "component": "inih", + "file": "ini.c", + "file_hash": "5f403e492ed8b5519bb53d937f772bc4", + "file_url": " ", + "hits": 120, + "id": "snippet", + "latest": "498f34b", + "licenses": [ + { + "checklist_url": "https://www.osadl.org/fileadmin/checklists/unreflicenses/BSD-3-Clause.txt", + "copyleft": "no", + "name": "BSD-3-Clause", + "osadl_updated": "2026-02-12T16:31:00+0000", + "patent_hints": "no", + "source": "license_file", + "url": "https://spdx.org/licenses/BSD-3-Clause.html" + } + ], + "lines": "7-324", + "matched": "94%", + "oss_lines": "7-324", + "purl": [ + "pkg:github/benhoyt/inih", + "pkg:apk/inih", + "pkg:rpm/opensuse/libinih-devel", + "pkg:generic/inih", + "pkg:golang/github.com/benhoyt/inih", + "pkg:conan/inih" + ], + "release_date": "2025-07-09", + "server": { + "kb_version": { + "daily": "26.10.19", + "monthly": "26.09" + }, + "version": "5.4.27" + }, + "source_hash": "6d05d0c61839b890ecb934a76a715fbe", + "status": "pending", + "url": "https://github.com/benhoyt/inih", + "url_hash": "6e29bc43eb4149a76fde63d41a5d26d2", + "vendor": "benhoyt", + "version": "498f34b" + } + ], + "no-origin/src/greenhouse_watering.py": [ + { + "id": "none", + "server": { + "kb_version": { + "daily": "26.10.19", + "monthly": "26.09" + }, + "version": "5.4.27" + } + } + ] +} diff --git a/scanpipe/tests/data/find-origins/scheduler.js b/scanpipe/tests/data/find-origins/scheduler.js new file mode 100644 index 0000000000..9538de4629 --- /dev/null +++ b/scanpipe/tests/data/find-origins/scheduler.js @@ -0,0 +1,162 @@ +/* + * Refresh scheduler for the dashboard widgets. + */ + +const FUNC_ERROR_TEXT = 'Expected a function'; +const nativeMax = Math.max; +const nativeMin = Math.min; + +function now() { + return Date.now(); +} + +function isObject(value) { + const valueType = typeof value; + return value != null && (valueType === 'object' || valueType === 'function'); +} + +function toNumber(value) { + return typeof value === 'number' ? value : Number(value); +} + +function debounce(func, wait, options) { + var lastArgs, + lastThis, + maxWait, + result, + timerId, + lastCallTime, + lastInvokeTime = 0, + leading = false, + maxing = false, + trailing = true; + + if (typeof func != 'function') { + throw new TypeError(FUNC_ERROR_TEXT); + } + wait = toNumber(wait) || 0; + if (isObject(options)) { + leading = !!options.leading; + maxing = 'maxWait' in options; + maxWait = maxing ? nativeMax(toNumber(options.maxWait) || 0, wait) : maxWait; + trailing = 'trailing' in options ? !!options.trailing : trailing; + } + + function invokeFunc(time) { + var args = lastArgs, + thisArg = lastThis; + + lastArgs = lastThis = undefined; + lastInvokeTime = time; + result = func.apply(thisArg, args); + return result; + } + + function leadingEdge(time) { + // Reset any `maxWait` timer. + lastInvokeTime = time; + // Start the timer for the trailing edge. + timerId = setTimeout(timerExpired, wait); + // Invoke the leading edge. + return leading ? invokeFunc(time) : result; + } + + function remainingWait(time) { + var timeSinceLastCall = time - lastCallTime, + timeSinceLastInvoke = time - lastInvokeTime, + timeWaiting = wait - timeSinceLastCall; + + return maxing + ? nativeMin(timeWaiting, maxWait - timeSinceLastInvoke) + : timeWaiting; + } + + function shouldInvoke(time) { + var timeSinceLastCall = time - lastCallTime, + timeSinceLastInvoke = time - lastInvokeTime; + + // Either this is the first call, activity has stopped and we're at the + // trailing edge, the system time has gone backwards and we're treating + // it as the trailing edge, or we've hit the `maxWait` limit. + return (lastCallTime === undefined || (timeSinceLastCall >= wait) || + (timeSinceLastCall < 0) || (maxing && timeSinceLastInvoke >= maxWait)); + } + + function timerExpired() { + var time = now(); + if (shouldInvoke(time)) { + return trailingEdge(time); + } + // Restart the timer. + timerId = setTimeout(timerExpired, remainingWait(time)); + } + + function trailingEdge(time) { + timerId = undefined; + + // Only invoke if we have `lastArgs` which means `func` has been + // debounced at least once. + if (trailing && lastArgs) { + return invokeFunc(time); + } + lastArgs = lastThis = undefined; + return result; + } + + function cancel() { + if (timerId !== undefined) { + clearTimeout(timerId); + } + lastInvokeTime = 0; + lastArgs = lastCallTime = lastThis = timerId = undefined; + } + + function flush() { + return timerId === undefined ? result : trailingEdge(now()); + } + + function debounced() { + var time = now(), + isInvoking = shouldInvoke(time); + + lastArgs = arguments; + lastThis = this; + lastCallTime = time; + + if (isInvoking) { + if (timerId === undefined) { + return leadingEdge(lastCallTime); + } + if (maxing) { + // Handle invocations in a tight loop. + clearTimeout(timerId); + timerId = setTimeout(timerExpired, wait); + return invokeFunc(lastCallTime); + } + } + if (timerId === undefined) { + timerId = setTimeout(timerExpired, wait); + } + return result; + } + debounced.cancel = cancel; + debounced.flush = flush; + return debounced; +} + +const pendingRefreshes = new Map(); + +function scheduleRefresh(widgetId, refresh, delayMs = 250) { + if (!pendingRefreshes.has(widgetId)) { + const debouncedRefresh = debounce(refresh, delayMs, { maxWait: delayMs * 4 }); + pendingRefreshes.set(widgetId, debouncedRefresh); + } + pendingRefreshes.get(widgetId)(); +} + +function cancelRefresh(widgetId) { + pendingRefreshes.get(widgetId)?.cancel(); + pendingRefreshes.delete(widgetId); +} + +module.exports = { scheduleRefresh, cancelRefresh }; diff --git a/scanpipe/tests/pipes/test_matchcode.py b/scanpipe/tests/pipes/test_matchcode.py index 1e46d85741..77e1c5499a 100644 --- a/scanpipe/tests/pipes/test_matchcode.py +++ b/scanpipe/tests/pipes/test_matchcode.py @@ -30,9 +30,11 @@ from scanpipe.models import AbstractTaskFieldsModel from scanpipe.models import CodebaseResource +from scanpipe.models import OriginMatch from scanpipe.models import Project from scanpipe.pipes import matchcode from scanpipe.pipes.input import copy_input +from scanpipe.tests import make_resource_directory from scanpipe.tests import make_resource_file @@ -42,6 +44,14 @@ class MatchCodePipesTest(TestCase): def setUp(self): self.project1 = Project.objects.create(name="Analysis") + def make_scheduler_resource(self, path): + """Create a resource at ``path`` with the scheduler.js fixture content.""" + resource = make_resource_file(self.project1, path) + resource.location_path.parent.mkdir(parents=True, exist_ok=True) + scheduler_location = self.data / "find-origins" / "scheduler.js" + resource.location_path.write_text(scheduler_location.read_text()) + return resource + def test_scanpipe_pipes_matchcode_fingerprint_codebase_directories(self): fixtures = self.data / "asgiref" / "asgiref-3.3.0_fixtures.json" call_command("loaddata", fixtures, **{"verbosity": 0}) @@ -441,3 +451,172 @@ def test_scanpipe_pipes_matchcode_fingerprint_stemmed_codebase_resources(self): self.assertEqual(expected_extra_data, codebase_resource3.extra_data) self.assertFalse(codebase_resource1.extra_data) self.assertFalse(codebase_resource2.extra_data) + + @mock.patch("scanpipe.pipes.matchcode.get_match_results") + @mock.patch("scanpipe.pipes.matchcode.poll_run_url_status") + @mock.patch("scanpipe.pipes.matchcode.send_project_json_to_matchcode") + @mock.patch("scanpipe.pipes.matchcode.fingerprint_codebase_directories") + @mock.patch("scanpipe.pipes.matchcode.fingerprint_codebase_resources") + def test_scanpipe_pipes_matchcode_match_codebase( + self, + mock_fingerprint_resources, + mock_fingerprint_directories, + mock_send_project_json, + mock_poll_run_url_status, + mock_get_match_results, + ): + mock_send_project_json.return_value = ("match_url", "run_url") + mock_poll_run_url_status.return_value = True + mock_get_match_results.return_value = {"files": []} + + match_results = matchcode.match_codebase(self.project1) + self.assertEqual({"files": []}, match_results) + mock_fingerprint_resources.assert_called_once_with(self.project1) + mock_fingerprint_directories.assert_called_once_with(self.project1) + mock_send_project_json.assert_called_once_with(self.project1) + mock_poll_run_url_status.assert_called_once_with("run_url") + mock_get_match_results.assert_called_once_with("match_url") + + def test_scanpipe_pipes_matchcode_save_match_results(self): + match_results = {"files": [{"path": "six.py"}]} + matchcode.save_match_results(self.project1, match_results) + + output_file = next(self.project1.output_path.glob("matchcode-*.json")) + self.assertEqual(match_results, json.loads(output_file.read_text())) + + def test_scanpipe_pipes_matchcode_get_resource_line_by_pos(self): + resource = make_resource_file(self.project1, "words.txt") + resource.location_path.write_text("alpha beta\ngamma\n\ndelta\n") + + expected = {0: 1, 1: 1, 2: 2, 3: 4} + self.assertEqual(expected, matchcode.get_resource_line_by_pos(resource)) + + def test_scanpipe_pipes_matchcode_get_snippet_line_ranges(self): + line_by_pos = {0: 1, 1: 1, 2: 2, 3: 2, 4: 3, 5: 4, 6: 5} + line_ranges = matchcode.get_snippet_line_ranges([0, 1, 2, 5, 6], line_by_pos) + self.assertEqual([(1, 2), (4, 5)], line_ranges) + + def test_scanpipe_pipes_matchcode_get_line_coverage(self): + line_by_pos = {0: 1, 1: 2, 2: 4, 3: 5} + self.assertEqual(50, matchcode.get_line_coverage([(1, 3)], line_by_pos)) + self.assertEqual(100, matchcode.get_line_coverage([(1, 5)], line_by_pos)) + + def test_scanpipe_pipes_matchcode_is_legal_file_path(self): + self.assertTrue(matchcode.is_legal_file_path("package/LICENSE")) + self.assertTrue(matchcode.is_legal_file_path("crate-1.0/LICENSE-MIT")) + self.assertTrue(matchcode.is_legal_file_path("COPYING.txt")) + self.assertTrue(matchcode.is_legal_file_path("docs/mit.license")) + self.assertFalse(matchcode.is_legal_file_path("package/debounce.js")) + + def test_scanpipe_pipes_matchcode_get_package_extra_data(self): + package_data = { + "purl": "pkg:npm/ms@2.1.3", + "package_uid": "pkg:npm/ms@2.1.3?uuid=1", + "release_date": None, + "declared_license_expression": "mit", + "download_url": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + } + expected = { + "package_uid": "pkg:npm/ms@2.1.3?uuid=1", + "release_date": "", + "declared_license_expression": "mit", + "download_url": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + } + self.assertEqual(expected, matchcode.get_package_extra_data(package_data)) + + def test_scanpipe_pipes_matchcode_make_snippet_origin_matches(self): + resource = self.make_scheduler_resource("scheduler.js") + matched_snippets = [ + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "package/debounce.js", + "similarity": 0.005, + "match_detections": [59, 60, 61], + }, + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "package/LICENSE", + "similarity": 0.028, + "match_detections": [0, 1, 2], + }, + ] + + origin_matches = list( + matchcode.make_snippet_origin_matches( + resource, matched_snippets, {}, "38.1.0" + ) + ) + self.assertEqual(1, len(origin_matches)) + origin_match = origin_matches[0] + self.assertEqual(OriginMatch.MatchType.SNIPPET, origin_match.match_type) + self.assertEqual("package/debounce.js", origin_match.origin_path) + self.assertEqual(0.005, origin_match.extra_data["snippet_similarity"]) + self.assertEqual("38.1.0", origin_match.tool_version) + + def test_scanpipe_pipes_matchcode_make_package_origin_matches(self): + resource = make_resource_file(self.project1, "index.js") + package_data_by_uid = { + "uid1": {"purl": "pkg:npm/ms@2.1.3", "package_uid": "uid1"}, + "uid2": {"purl": "pkg:npm/ms@2.1.3", "package_uid": "uid2"}, + } + + origin_matches = list( + matchcode.make_package_origin_matches( + resource, + OriginMatch.MatchType.DIRECTORY, + ["uid1", "uid2"], + package_data_by_uid, + "38.1.0", + ) + ) + self.assertEqual(1, len(origin_matches)) + self.assertEqual("pkg:npm/ms@2.1.3", origin_matches[0].purl) + self.assertIsNone(origin_matches[0].similarity) + + origin_matches = list( + matchcode.make_package_origin_matches( + resource, + OriginMatch.MatchType.FILE, + ["uid1"], + package_data_by_uid, + "38.1.0", + ) + ) + self.assertEqual(100, origin_matches[0].similarity) + + def test_scanpipe_pipes_matchcode_load_matchcode_results(self): + results_location = self.data / "find-origins" / "matchcode-results.json" + match_results = json.loads(results_location.read_text()) + make_resource_directory(self.project1, "embedded-package/vendor/ms") + make_resource_file(self.project1, "embedded-package/vendor/ms/index.js") + make_resource_file(self.project1, "exact-copy/myapp/_vendor/six.py") + make_resource_file(self.project1, "no-origin/src/greenhouse_watering.py") + self.make_scheduler_resource("snippet/src/scheduler.js") + + origin_match_count = matchcode.load_matchcode_results( + self.project1, match_results + ) + self.assertEqual(5, origin_match_count) + + origin_matches = self.project1.originmatchs.order_by("resource__path", "purl") + expected = [ + ("embedded-package/vendor/ms", "pkg:npm/ms@2.1.3", "directory", None), + ( + "embedded-package/vendor/ms/index.js", + "pkg:npm/ms@2.1.3", + "directory", + None, + ), + ("exact-copy/myapp/_vendor/six.py", "pkg:pypi/six@1.17.0", "file", 100), + ("snippet/src/scheduler.js", "pkg:npm/lodash@4.17.21", "snippet", 75), + ("snippet/src/scheduler.js", "pkg:npm/lodash@4.17.21", "snippet", 75), + ] + values = origin_matches.values_list( + "resource__path", "purl", "match_type", "similarity" + ) + self.assertEqual(expected, list(values)) + + snippet_origin_match = origin_matches.get(origin_path__endswith="debounce.js") + self.assertEqual([{"lines": [26, 140]}], snippet_origin_match.line_ranges) + self.assertEqual("matchcode", snippet_origin_match.tool) + self.assertEqual("38.1.0", snippet_origin_match.tool_version) diff --git a/scanpipe/tests/pipes/test_pipes.py b/scanpipe/tests/pipes/test_pipes.py index ee0ab3a88f..7ea3f27107 100644 --- a/scanpipe/tests/pipes/test_pipes.py +++ b/scanpipe/tests/pipes/test_pipes.py @@ -22,6 +22,7 @@ import datetime import io +import subprocess from pathlib import Path from unittest import mock @@ -65,6 +66,20 @@ def test_scanpipe_pipes_strip_root(self): self.assertEqual(expected, pipes.strip_root(path)) self.assertEqual(expected, pipes.strip_root(Path(path))) + def test_scanpipe_pipes_run_command_safely_env(self): + output = pipes.run_command_safely( + ["/usr/bin/env"], env={"ORIGIN_TOOL_API_KEY": "secret-value"} + ) + self.assertEqual("ORIGIN_TOOL_API_KEY=secret-value\n", output) + + with self.assertRaises(subprocess.SubprocessError) as context: + pipes.run_command_safely( + ["/usr/bin/env", "missing-command"], + env={"ORIGIN_TOOL_API_KEY": "secret-value"}, + ) + self.assertIn("missing-command", str(context.exception)) + self.assertNotIn("secret-value", str(context.exception)) + @mock.patch("scanpipe.pipes.datetime", mocked_now) def test_scanpipe_pipes_filename_now(self): self.assertEqual("2010-10-10-10-10-10", pipes.filename_now()) diff --git a/scanpipe/tests/pipes/test_scanoss.py b/scanpipe/tests/pipes/test_scanoss.py new file mode 100644 index 0000000000..1f92d65a0a --- /dev/null +++ b/scanpipe/tests/pipes/test_scanoss.py @@ -0,0 +1,187 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/nexB/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/nexB/scancode.io for support and download. + +import json +from pathlib import Path +from unittest import mock + +from django.test import TestCase + +from scanpipe.models import OriginMatch +from scanpipe.pipes import scanoss +from scanpipe.tests import make_project +from scanpipe.tests import make_resource_file + + +class ScanOSSPipesTest(TestCase): + data = Path(__file__).parent.parent / "data" / "find-origins" + + def setUp(self): + self.project1 = make_project() + scanoss_results_location = self.data / "scanoss-results.json" + self.scanoss_results = json.loads(scanoss_results_location.read_text()) + + def test_scanpipe_pipes_scanoss_is_configured(self): + with mock.patch.object(scanoss, "SCANOSS_URL", ""): + self.assertFalse(scanoss.is_configured()) + + with mock.patch.object(scanoss, "SCANOSS_URL", "https://api.osskb.org"): + self.assertTrue(scanoss.is_configured()) + + @mock.patch("shutil.which") + def test_scanpipe_pipes_scanoss_get_uvx_location(self, mock_which): + mock_which.return_value = "/usr/bin/uvx" + self.assertEqual("/usr/bin/uvx", scanoss.get_uvx_location()) + mock_which.assert_called_once_with("uvx") + + @mock.patch("scanpipe.pipes.scanoss.run_command_safely") + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") + def test_scanpipe_pipes_scanoss_scan_codebase( + self, mock_get_uvx_location, mock_run_command_safely + ): + mock_get_uvx_location.return_value = "/usr/bin/uvx" + mock_run_command_safely.return_value = "" + + with ( + mock.patch.object(scanoss, "SCANOSS_URL", "https://api.osskb.org"), + mock.patch.object(scanoss, "SCANOSS_API_KEY", "secret-key"), + ): + scanoss.scan_codebase("/codebase", "/output/scanoss.json") + + command_args = mock_run_command_safely.call_args.args[0] + scanoss_env = mock_run_command_safely.call_args.kwargs["env"] + self.assertEqual("/usr/bin/uvx", command_args[0]) + self.assertIn(f"--from=scanoss=={scanoss.SCANOSS_PY_VERSION}", command_args) + self.assertIn("--apiurl=https://api.osskb.org", command_args) + self.assertIn("--flags=256", command_args) + self.assertEqual( + ["--output=/output/scanoss.json", "/codebase"], command_args[-2:] + ) + self.assertNotIn("secret-key", " ".join(command_args)) + self.assertEqual(["HOME", "PATH", "SCANOSS_API_KEY"], sorted(scanoss_env)) + self.assertEqual("secret-key", scanoss_env["SCANOSS_API_KEY"]) + + def test_scanpipe_pipes_scanoss_to_line_range(self): + self.assertEqual([40, 326], scanoss.to_line_range("40-326")) + + def test_scanpipe_pipes_scanoss_parse_line_ranges(self): + self.assertEqual([], scanoss.parse_line_ranges("all", "all")) + + expected = [ + {"lines": [1, 37], "origin_lines": [5, 41]}, + {"lines": [40, 326], "origin_lines": [44, 330]}, + ] + line_ranges = scanoss.parse_line_ranges("1-37,40-326", "5-41,44-330") + self.assertEqual(expected, line_ranges) + + def test_scanpipe_pipes_scanoss_parse_similarity(self): + self.assertEqual(94, scanoss.parse_similarity("94%")) + + def test_scanpipe_pipes_scanoss_get_versioned_purl(self): + component = {"purl": ["pkg:github/benjaminp/six", "pkg:pypi/six"]} + component["version"] = "1.17.0" + self.assertEqual( + "pkg:github/benjaminp/six@1.17.0", scanoss.get_versioned_purl(component) + ) + + component = {"purl": ["pkg:pypi/six@1.16.0"], "version": "1.17.0"} + self.assertEqual("pkg:pypi/six@1.16.0", scanoss.get_versioned_purl(component)) + + self.assertEqual("", scanoss.get_versioned_purl({"version": "1.0"})) + + component = {"purl": ["not-a-purl"], "version": "1.0"} + self.assertEqual("", scanoss.get_versioned_purl(component)) + + def test_scanpipe_pipes_scanoss_get_components(self): + best_match = {"id": "file", "purl": ["pkg:pypi/six"]} + self.assertEqual([best_match], scanoss.get_components(best_match)) + + components = [{"purl": ["pkg:pypi/six"]}, {"purl": ["pkg:pypi/urllib3"]}] + scanoss_match = {"id": "file", "components": components} + self.assertEqual(components, scanoss.get_components(scanoss_match)) + + def test_scanpipe_pipes_scanoss_get_origin_match_extra_data(self): + scanoss_match = self.scanoss_results["exact-copy/myapp/_vendor/six.py"][0] + extra_data = scanoss.get_origin_match_extra_data(scanoss_match, scanoss_match) + + expected = { + "purls": [ + "pkg:github/benjaminp/six", + "pkg:pypi/six", + "pkg:golang/github.com/benjaminp/six", + "pkg:conda/six", + ], + "release_date": "2024-12-03", + "url": "https://github.com/benjaminp/six", + "licenses": [{"name": "MIT", "source": "component_declared"}], + "file_hash": "71e8581c332473a17735159487901fb5", + "source_hash": "71e8581c332473a17735159487901fb5", + "hits": None, + } + self.assertEqual(expected, extra_data) + + def test_scanpipe_pipes_scanoss_make_origin_matches(self): + resource = make_resource_file(self.project1, "modified-copy/src/inih/ini.c") + scanoss_matches = self.scanoss_results["modified-copy/src/inih/ini.c"] + + origin_matches = list(scanoss.make_origin_matches(resource, scanoss_matches)) + self.assertEqual(1, len(origin_matches)) + origin_match = origin_matches[0] + self.assertEqual("pkg:github/benhoyt/inih@498f34b", origin_match.purl) + self.assertEqual(OriginMatch.MatchType.SNIPPET, origin_match.match_type) + self.assertEqual("scanoss", origin_match.tool) + self.assertEqual("5.4.27", origin_match.tool_version) + self.assertEqual("ini.c", origin_match.origin_path) + expected_line_ranges = [{"lines": [7, 324], "origin_lines": [7, 324]}] + self.assertEqual(expected_line_ranges, origin_match.line_ranges) + self.assertEqual(94, origin_match.similarity) + + no_match = self.scanoss_results["no-origin/src/greenhouse_watering.py"] + self.assertEqual([], list(scanoss.make_origin_matches(resource, no_match))) + + def test_scanpipe_pipes_scanoss_load_scanoss_results(self): + six_resource = make_resource_file( + self.project1, "exact-copy/myapp/_vendor/six.py" + ) + make_resource_file(self.project1, "modified-copy/src/inih/ini.c") + results_location = self.data / "scanoss-results.json" + + origin_match_count = scanoss.load_scanoss_results( + self.project1, results_location + ) + self.assertEqual(2, origin_match_count) + six_origin_match = six_resource.origin_matches.get() + self.assertEqual("pkg:github/benjaminp/six@1.17.0", six_origin_match.purl) + self.assertEqual(OriginMatch.MatchType.FILE, six_origin_match.match_type) + self.assertEqual(100, six_origin_match.similarity) + self.assertEqual([], six_origin_match.line_ranges) + + message = self.project1.projectmessages.get() + self.assertEqual( + "SCANOSS results for paths missing from the codebase", message.description + ) + self.assertEqual( + {"paths": ["no-origin/src/greenhouse_watering.py"]}, message.details + ) + + scanoss.load_scanoss_results(self.project1, results_location) + self.assertEqual(2, self.project1.originmatchs.count()) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 442fdea4df..f805f7ed3d 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -47,6 +47,7 @@ from scanpipe.pipelines import deploy_to_develop from scanpipe.pipelines import is_pipeline from scanpipe.pipelines import scan_single_package +from scanpipe.pipelines.find_origins import FindOrigins from scanpipe.pipelines.find_vulnerabilities import FindVulnerabilities from scanpipe.pipes import d2d from scanpipe.pipes import flag @@ -58,6 +59,7 @@ from scanpipe.tests import make_mock_response from scanpipe.tests import make_package from scanpipe.tests import make_project +from scanpipe.tests import make_resource_file from scanpipe.tests import package_data1 from scanpipe.tests.pipelines.do_nothing import DoNothing from scanpipe.tests.pipelines.download_inputs import DownloadInput @@ -161,6 +163,21 @@ def test_scanpipe_find_vulnerabilities_get_info_availability(self): self.assertTrue(info["is_available"]) self.assertEqual("", info["unavailable_reason"]) + @mock.patch("scanpipe.pipes.scanoss.is_configured") + @mock.patch("scanpipe.pipes.matchcode.is_configured") + def test_scanpipe_find_origins_get_availability( + self, mock_matchcode_is_configured, mock_scanoss_is_configured + ): + mock_matchcode_is_configured.return_value = False + mock_scanoss_is_configured.return_value = False + self.assertEqual( + "No origin detection tool is configured: MatchCode.io or SCANOSS.", + FindOrigins.get_availability(), + ) + + mock_scanoss_is_configured.return_value = True + self.assertIsNone(FindOrigins.get_availability()) + def test_scanpipe_pipeline_class_get_summary(self): expected = "Do nothing, in 2 steps." self.assertEqual(expected, DoNothing.get_summary()) @@ -1512,6 +1529,95 @@ def test_scanpipe_fetch_scores_pipeline_integration(self, mock_is_available): self.assertEqual("v5.2.1", scorecard_entry.scoring_tool_version) self.assertTrue(scorecard_entry.score) + @mock.patch("scanpipe.pipes.scanoss.scan_codebase") + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") + @mock.patch("scanpipe.pipes.scanoss.is_configured") + @mock.patch("scanpipe.pipes.matchcode.match_codebase") + @mock.patch("scanpipe.pipes.matchcode.is_configured") + def test_scanpipe_find_origins_pipeline_integration( + self, + mock_matchcode_is_configured, + mock_match_codebase, + mock_scanoss_is_configured, + mock_get_uvx_location, + mock_scan_codebase, + ): + pipeline_name = "find_origins" + project1 = make_project() + six_path = "exact-copy/myapp/_vendor/six.py" + make_resource_file(project1, six_path) + make_resource_file(project1, "modified-copy/src/inih/ini.c") + find_origins_data = self.data / "find-origins" + + match_results = json.loads( + (find_origins_data / "matchcode-results.json").read_text() + ) + match_results["files"] = [ + resource + for resource in match_results["files"] + if resource["path"] == six_path + ] + mock_matchcode_is_configured.return_value = True + mock_match_codebase.return_value = match_results + + def write_scanoss_results(codebase_path, output_location): + scanoss_results = (find_origins_data / "scanoss-results.json").read_text() + Path(output_location).write_text(scanoss_results) + + mock_scanoss_is_configured.return_value = True + mock_get_uvx_location.return_value = "/usr/bin/uvx" + mock_scan_codebase.side_effect = write_scanoss_results + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + origin_matches = project1.originmatchs.order_by("tool", "resource__path") + expected = [ + (six_path, "pkg:pypi/six@1.17.0", "matchcode"), + (six_path, "pkg:github/benjaminp/six@1.17.0", "scanoss"), + ( + "modified-copy/src/inih/ini.c", + "pkg:github/benhoyt/inih@498f34b", + "scanoss", + ), + ] + values = origin_matches.values_list("resource__path", "purl", "tool") + self.assertEqual(expected, list(values)) + self.assertEqual(1, len(list(project1.output_path.glob("matchcode-*.json")))) + + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") + @mock.patch("scanpipe.pipes.scanoss.is_configured") + @mock.patch("scanpipe.pipes.matchcode.is_configured") + def test_scanpipe_find_origins_pipeline_skip_unavailable_tools( + self, + mock_matchcode_is_configured, + mock_scanoss_is_configured, + mock_get_uvx_location, + ): + pipeline_name = "find_origins" + project1 = make_project() + mock_matchcode_is_configured.return_value = False + mock_scanoss_is_configured.return_value = True + mock_get_uvx_location.return_value = None + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + run.refresh_from_db() + self.assertIn("MatchCode.io is not configured, skipping.", run.log) + uvx_missing_message = ( + "The uvx executable required by SCANOSS is missing, skipping." + ) + self.assertIn(uvx_missing_message, run.log) + message = project1.projectmessages.get() + self.assertEqual(uvx_missing_message, message.description) + self.assertEqual("find_scanoss_origin_matches", message.model) + self.assertEqual(0, project1.originmatchs.count()) + def test_scanpipe_resolve_dependencies_pipeline_integration(self): pipeline_name = "resolve_dependencies" project1 = make_project() From a4d4a8779caecf1f81ea855d70b1497f36cfe285 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 20 Oct 2026 09:07:48 +0400 Subject: [PATCH 11/12] record origin tool failures, batch origin match creation, and add missing tests #1926 Signed-off-by: tdruez --- scanpipe/pipelines/find_origins.py | 27 +++++++----- scanpipe/pipes/__init__.py | 35 +++++++++++++++ scanpipe/pipes/matchcode.py | 49 ++++++++++----------- scanpipe/pipes/scanoss.py | 17 +++---- scanpipe/tests/pipes/test_matchcode.py | 61 ++++++++++++++++++++++++++ scanpipe/tests/pipes/test_pipes.py | 31 +++++++++++++ scanpipe/tests/test_pipelines.py | 39 ++++++++++++++++ 7 files changed, 211 insertions(+), 48 deletions(-) diff --git a/scanpipe/pipelines/find_origins.py b/scanpipe/pipelines/find_origins.py index a43e25c3f7..b37e67c5e6 100644 --- a/scanpipe/pipelines/find_origins.py +++ b/scanpipe/pipelines/find_origins.py @@ -20,6 +20,8 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. +import subprocess + from scanpipe.pipelines import Pipeline from scanpipe.pipes import matchcode from scanpipe.pipes import scanoss @@ -62,13 +64,15 @@ def find_matchcode_origin_matches(self): self.log("MatchCode.io is not configured, skipping.") return - match_results = matchcode.match_codebase(self.project) - matchcode.save_match_results(self.project, match_results) + # A tool failure is recorded and does not prevent running the other tools. + with self.save_errors(matchcode.MatchCodeIOException): + match_results = matchcode.match_codebase(self.project) + matchcode.save_match_results(self.project, match_results) - origin_match_count = matchcode.load_matchcode_results( - self.project, match_results - ) - self.log(f"{origin_match_count:,d} MatchCode.io origin matches processed.") + origin_match_count = matchcode.load_matchcode_results( + self.project, match_results + ) + self.log(f"{origin_match_count:,d} MatchCode.io origin matches processed.") def find_scanoss_origin_matches(self): """Scan the codebase with SCANOSS and store the reported origin matches.""" @@ -85,9 +89,10 @@ def find_scanoss_origin_matches(self): return results_location = self.project.get_output_file_path("scanoss", "json") - scanoss.scan_codebase(self.project.codebase_path, results_location) + with self.save_errors(subprocess.SubprocessError): + scanoss.scan_codebase(self.project.codebase_path, results_location) - origin_match_count = scanoss.load_scanoss_results( - self.project, results_location - ) - self.log(f"{origin_match_count:,d} SCANOSS origin matches processed.") + origin_match_count = scanoss.load_scanoss_results( + self.project, results_location + ) + self.log(f"{origin_match_count:,d} SCANOSS origin matches processed.") diff --git a/scanpipe/pipes/__init__.py b/scanpipe/pipes/__init__.py index 8e77d92fae..62e932071c 100644 --- a/scanpipe/pipes/__init__.py +++ b/scanpipe/pipes/__init__.py @@ -39,6 +39,7 @@ from scanpipe.models import DiscoveredDependency from scanpipe.models import DiscoveredLicense from scanpipe.models import DiscoveredPackage +from scanpipe.models import OriginMatch from scanpipe.pipes import scancode logger = logging.getLogger("scanpipe.pipes") @@ -615,3 +616,37 @@ def run_command_safely(command_args, env=None): raise subprocess.SubprocessError(error_msg) return completed_process.stdout + + +def create_origin_matches( + project, results_by_path, make_origin_matches, batch_size=2000 +): + """ + Create OriginMatch records for the ``project`` resources from the + ``results_by_path`` mapping of resource path to origin tool results. + The ``make_origin_matches(resource, results)`` function yields the unsaved + OriginMatch instances of a resource. Resources are processed by batches of + ``batch_size`` paths. + Return the number of processed origin matches and the list of paths missing + from the project codebase. + """ + paths = iter(results_by_path) + origin_match_count = 0 + missing_paths = [] + + while batch_paths := list(islice(paths, batch_size)): + resources = project.codebaseresources.filter(path__in=batch_paths) + resources_by_path = {resource.path: resource for resource in resources} + missing_paths.extend( + path for path in batch_paths if path not in resources_by_path + ) + + origin_matches = [ + origin_match + for path, resource in resources_by_path.items() + for origin_match in make_origin_matches(resource, results_by_path[path]) + ] + OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) + origin_match_count += len(origin_matches) + + return origin_match_count, missing_paths diff --git a/scanpipe/pipes/matchcode.py b/scanpipe/pipes/matchcode.py index f7bef0773c..690e834ded 100644 --- a/scanpipe/pipes/matchcode.py +++ b/scanpipe/pipes/matchcode.py @@ -23,6 +23,7 @@ import json import logging from collections import defaultdict +from functools import partial from pathlib import Path from django.conf import settings @@ -39,6 +40,7 @@ from scanpipe.models import OriginMatch from scanpipe.pipes import codebase +from scanpipe.pipes import create_origin_matches from scanpipe.pipes import flag from scanpipe.pipes import poll_until_success from scanpipe.pipes.output import to_json @@ -450,7 +452,11 @@ def match_codebase(project): fingerprint_codebase_directories(project) match_url, run_url = send_project_json_to_matchcode(project) poll_run_url_status(run_url) - return get_match_results(match_url) + + match_results = get_match_results(match_url) + if not match_results: + raise MatchCodeIOException("Cannot fetch the match results from MatchCode.io") + return match_results def save_match_results(project, match_results): @@ -461,8 +467,7 @@ def save_match_results(project, match_results): def get_resource_line_by_pos(resource): """Return the mapping of token positions to line numbers of a ``resource``.""" - with open(resource.location) as resource_file: - return get_line_by_pos(resource_file.read()) + return get_line_by_pos(resource.location_path.read_text()) def get_snippet_line_ranges(match_detections, line_by_pos): @@ -492,7 +497,7 @@ def get_line_coverage(line_ranges, line_by_pos): def is_legal_file_path(path): - """Return True if the file name of ``path`` is the one of a legal file.""" + """Return True if ``path`` is a legal file, such as a license or a notice.""" file_path = Path(path.lower()) return ( file_path.name.startswith(LEGAL_STARTS_ENDS) @@ -620,27 +625,19 @@ def load_matchcode_results(project, match_results): package_data_by_purl = { package_data["purl"]: package_data for package_data in packages_data } - tool_version = match_results["headers"][0]["tool_version"] - - resources_results = [ - resource_results + results_by_path = { + resource_results["path"]: resource_results for resource_results in match_results.get("files", []) if resource_results.get("for_packages") - ] - matched_paths = [resource_results["path"] for resource_results in resources_results] - resources = project.codebaseresources.filter(path__in=matched_paths) - resources_by_path = {resource.path: resource for resource in resources} - - origin_matches = [ - origin_match - for resource_results in resources_results - for origin_match in make_origin_matches( - resources_by_path[resource_results["path"]], - resource_results, - package_data_by_uid, - package_data_by_purl, - tool_version, - ) - ] - OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) - return len(origin_matches) + } + make_matchcode_origin_matches = partial( + make_origin_matches, + package_data_by_uid=package_data_by_uid, + package_data_by_purl=package_data_by_purl, + tool_version=match_results["headers"][0]["tool_version"], + ) + + origin_match_count, _ = create_origin_matches( + project, results_by_path, make_matchcode_origin_matches + ) + return origin_match_count diff --git a/scanpipe/pipes/scanoss.py b/scanpipe/pipes/scanoss.py index f00d09091c..ddcc2a1d8c 100644 --- a/scanpipe/pipes/scanoss.py +++ b/scanpipe/pipes/scanoss.py @@ -31,6 +31,7 @@ from packageurl import PackageURL from scanpipe.models import OriginMatch +from scanpipe.pipes import create_origin_matches from scanpipe.pipes import run_command_safely TOOL_NAME = "scanoss" @@ -208,21 +209,15 @@ def load_scanoss_results(project, results_location): SCANOSS matches. Return the number of processed origin matches. """ scanoss_results = json.loads(Path(results_location).read_text()) - resources = project.codebaseresources.filter(path__in=scanoss_results.keys()) - resources_by_path = {resource.path: resource for resource in resources} + origin_match_count, missing_paths = create_origin_matches( + project, scanoss_results, make_origin_matches + ) - missing_paths = sorted(set(scanoss_results) - set(resources_by_path)) if missing_paths: project.add_warning( description="SCANOSS results for paths missing from the codebase", model="OriginMatch", - details={"paths": missing_paths}, + details={"paths": sorted(missing_paths)}, ) - origin_matches = [ - origin_match - for path, resource in resources_by_path.items() - for origin_match in make_origin_matches(resource, scanoss_results[path]) - ] - OriginMatch.objects.bulk_create(origin_matches, ignore_conflicts=True) - return len(origin_matches) + return origin_match_count diff --git a/scanpipe/tests/pipes/test_matchcode.py b/scanpipe/tests/pipes/test_matchcode.py index 77e1c5499a..a00748380b 100644 --- a/scanpipe/tests/pipes/test_matchcode.py +++ b/scanpipe/tests/pipes/test_matchcode.py @@ -477,6 +477,10 @@ def test_scanpipe_pipes_matchcode_match_codebase( mock_poll_run_url_status.assert_called_once_with("run_url") mock_get_match_results.assert_called_once_with("match_url") + mock_get_match_results.return_value = None + with self.assertRaises(matchcode.MatchCodeIOException): + matchcode.match_codebase(self.project1) + def test_scanpipe_pipes_matchcode_save_match_results(self): match_results = {"files": [{"path": "six.py"}]} matchcode.save_match_results(self.project1, match_results) @@ -584,6 +588,63 @@ def test_scanpipe_pipes_matchcode_make_package_origin_matches(self): ) self.assertEqual(100, origin_matches[0].similarity) + def test_scanpipe_pipes_matchcode_make_origin_matches(self): + resource = self.make_scheduler_resource("scheduler.js") + package_data = {"purl": "pkg:npm/lodash@4.17.21", "package_uid": "uid1"} + package_data_by_uid = {"uid1": package_data} + package_data_by_purl = {"pkg:npm/lodash@4.17.21": package_data} + + file_results = { + "status": "matched-to-purldb-resource", + "for_packages": ["uid1"], + "extra_data": {}, + } + origin_matches = list( + matchcode.make_origin_matches( + resource, + file_results, + package_data_by_uid, + package_data_by_purl, + "38.1.0", + ) + ) + self.assertEqual(OriginMatch.MatchType.FILE, origin_matches[0].match_type) + + snippet_results = { + "status": "scanned", + "for_packages": ["uid1"], + "extra_data": { + "matched_snippets": [ + { + "package": "pkg:npm/lodash@4.17.21", + "resource": "package/debounce.js", + "similarity": 0.005, + "match_detections": [59, 60, 61], + } + ] + }, + } + origin_matches = list( + matchcode.make_origin_matches( + resource, + snippet_results, + package_data_by_uid, + package_data_by_purl, + "38.1.0", + ) + ) + self.assertEqual(OriginMatch.MatchType.SNIPPET, origin_matches[0].match_type) + + unknown_status_results = {**file_results, "status": "scanned"} + origin_matches = matchcode.make_origin_matches( + resource, + unknown_status_results, + package_data_by_uid, + package_data_by_purl, + "38.1.0", + ) + self.assertEqual([], list(origin_matches)) + def test_scanpipe_pipes_matchcode_load_matchcode_results(self): results_location = self.data / "find-origins" / "matchcode-results.json" match_results = json.loads(results_location.read_text()) diff --git a/scanpipe/tests/pipes/test_pipes.py b/scanpipe/tests/pipes/test_pipes.py index 7ea3f27107..45ae5e5ad1 100644 --- a/scanpipe/tests/pipes/test_pipes.py +++ b/scanpipe/tests/pipes/test_pipes.py @@ -33,6 +33,7 @@ from scanpipe import pipes from scanpipe.models import CodebaseResource from scanpipe.models import DiscoveredPackage +from scanpipe.models import OriginMatch from scanpipe.models import Project from scanpipe.pipes import flag from scanpipe.pipes import get_resource_diff_ratio @@ -80,6 +81,36 @@ def test_scanpipe_pipes_run_command_safely_env(self): self.assertIn("missing-command", str(context.exception)) self.assertNotIn("secret-value", str(context.exception)) + def test_scanpipe_pipes_create_origin_matches(self): + project1 = make_project() + six_resource = make_resource_file(project1, "six.py") + ini_resource = make_resource_file(project1, "ini.c") + results_by_path = { + "six.py": ["pkg:pypi/six@1.17.0"], + "missing.py": ["pkg:pypi/missing@1.0"], + "ini.c": ["pkg:github/benhoyt/inih@r62"], + } + + def make_origin_matches(resource, purls): + for purl in purls: + yield OriginMatch( + project=project1, + resource=resource, + purl=purl, + match_type=OriginMatch.MatchType.FILE, + tool="test", + ) + + origin_match_count, missing_paths = pipes.create_origin_matches( + project1, results_by_path, make_origin_matches, batch_size=2 + ) + self.assertEqual(2, origin_match_count) + self.assertEqual(["missing.py"], missing_paths) + self.assertEqual("pkg:pypi/six@1.17.0", six_resource.origin_matches.get().purl) + self.assertEqual( + "pkg:github/benhoyt/inih@r62", ini_resource.origin_matches.get().purl + ) + @mock.patch("scanpipe.pipes.datetime", mocked_now) def test_scanpipe_pipes_filename_now(self): self.assertEqual("2010-10-10-10-10-10", pipes.filename_now()) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index f805f7ed3d..0572b77097 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -23,6 +23,7 @@ import io import json import os +import subprocess import sys import tempfile from contextlib import redirect_stderr @@ -51,6 +52,7 @@ from scanpipe.pipelines.find_vulnerabilities import FindVulnerabilities from scanpipe.pipes import d2d from scanpipe.pipes import flag +from scanpipe.pipes import matchcode from scanpipe.pipes import output from scanpipe.pipes import scancode from scanpipe.pipes.input import copy_input @@ -1587,6 +1589,43 @@ def write_scanoss_results(codebase_path, output_location): self.assertEqual(expected, list(values)) self.assertEqual(1, len(list(project1.output_path.glob("matchcode-*.json")))) + @mock.patch("scanpipe.pipes.scanoss.scan_codebase") + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") + @mock.patch("scanpipe.pipes.scanoss.is_configured") + @mock.patch("scanpipe.pipes.matchcode.match_codebase") + @mock.patch("scanpipe.pipes.matchcode.is_configured") + def test_scanpipe_find_origins_pipeline_record_tool_failures( + self, + mock_matchcode_is_configured, + mock_match_codebase, + mock_scanoss_is_configured, + mock_get_uvx_location, + mock_scan_codebase, + ): + pipeline_name = "find_origins" + project1 = make_project() + mock_matchcode_is_configured.return_value = True + mock_match_codebase.side_effect = matchcode.MatchCodeIOException( + "Cannot fetch the match results from MatchCode.io" + ) + mock_scanoss_is_configured.return_value = True + mock_get_uvx_location.return_value = "/usr/bin/uvx" + mock_scan_codebase.side_effect = subprocess.SubprocessError("SCANOSS failed") + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + error_descriptions = project1.projectmessages.values_list( + "description", flat=True + ) + expected = [ + "Cannot fetch the match results from MatchCode.io", + "SCANOSS failed", + ] + self.assertEqual(expected, sorted(error_descriptions)) + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") @mock.patch("scanpipe.pipes.scanoss.is_configured") @mock.patch("scanpipe.pipes.matchcode.is_configured") From de5394c9e92af3843fc17e37cb58992c0d00c9b2 Mon Sep 17 00:00:00 2001 From: tdruez Date: Tue, 20 Oct 2026 09:33:34 +0400 Subject: [PATCH 12/12] keep the SCANOSS knowledge base version and record invalid SCANOSS results #1926 Signed-off-by: tdruez --- scanpipe/models.py | 3 ++- scanpipe/pipelines/find_origins.py | 3 ++- scanpipe/pipes/scanoss.py | 1 + scanpipe/tests/pipes/test_scanoss.py | 1 + scanpipe/tests/test_pipelines.py | 27 +++++++++++++++++++++++++++ 5 files changed, 33 insertions(+), 2 deletions(-) diff --git a/scanpipe/models.py b/scanpipe/models.py index 343a672cab..a5f051f038 100644 --- a/scanpipe/models.py +++ b/scanpipe/models.py @@ -5110,7 +5110,8 @@ class MatchType(models.TextChoices): blank=True, help_text=_( "Matched line ranges, as a list of mappings with the 'lines' of the " - "resource and the corresponding 'origin_lines' of the origin file." + "resource and, when the tool reports them, the corresponding " + "'origin_lines' of the origin file." ), ) similarity = models.PositiveSmallIntegerField( diff --git a/scanpipe/pipelines/find_origins.py b/scanpipe/pipelines/find_origins.py index b37e67c5e6..815e2b34e1 100644 --- a/scanpipe/pipelines/find_origins.py +++ b/scanpipe/pipelines/find_origins.py @@ -89,7 +89,8 @@ def find_scanoss_origin_matches(self): return results_location = self.project.get_output_file_path("scanoss", "json") - with self.save_errors(subprocess.SubprocessError): + # A missing or invalid results file is recorded as a SCANOSS failure. + with self.save_errors(subprocess.SubprocessError, OSError, ValueError): scanoss.scan_codebase(self.project.codebase_path, results_location) origin_match_count = scanoss.load_scanoss_results( diff --git a/scanpipe/pipes/scanoss.py b/scanpipe/pipes/scanoss.py index ddcc2a1d8c..c542031c83 100644 --- a/scanpipe/pipes/scanoss.py +++ b/scanpipe/pipes/scanoss.py @@ -167,6 +167,7 @@ def get_origin_match_extra_data(scanoss_match, component): "file_hash": scanoss_match.get("file_hash", ""), "source_hash": scanoss_match.get("source_hash", ""), "hits": scanoss_match.get("hits"), + "kb_version": scanoss_match.get("server", {}).get("kb_version", {}), } diff --git a/scanpipe/tests/pipes/test_scanoss.py b/scanpipe/tests/pipes/test_scanoss.py index 1f92d65a0a..142fbc74f8 100644 --- a/scanpipe/tests/pipes/test_scanoss.py +++ b/scanpipe/tests/pipes/test_scanoss.py @@ -136,6 +136,7 @@ def test_scanpipe_pipes_scanoss_get_origin_match_extra_data(self): "file_hash": "71e8581c332473a17735159487901fb5", "source_hash": "71e8581c332473a17735159487901fb5", "hits": None, + "kb_version": {"daily": "26.10.19", "monthly": "26.09"}, } self.assertEqual(expected, extra_data) diff --git a/scanpipe/tests/test_pipelines.py b/scanpipe/tests/test_pipelines.py index 0572b77097..17d5ddfb4c 100644 --- a/scanpipe/tests/test_pipelines.py +++ b/scanpipe/tests/test_pipelines.py @@ -1626,6 +1626,33 @@ def test_scanpipe_find_origins_pipeline_record_tool_failures( ] self.assertEqual(expected, sorted(error_descriptions)) + @mock.patch("scanpipe.pipes.scanoss.scan_codebase") + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") + @mock.patch("scanpipe.pipes.scanoss.is_configured") + @mock.patch("scanpipe.pipes.matchcode.is_configured") + def test_scanpipe_find_origins_pipeline_record_missing_scanoss_results( + self, + mock_matchcode_is_configured, + mock_scanoss_is_configured, + mock_get_uvx_location, + mock_scan_codebase, + ): + pipeline_name = "find_origins" + project1 = make_project() + mock_matchcode_is_configured.return_value = False + mock_scanoss_is_configured.return_value = True + mock_get_uvx_location.return_value = "/usr/bin/uvx" + mock_scan_codebase.return_value = None + + run = project1.add_pipeline(pipeline_name) + pipeline = run.make_pipeline_instance() + exitcode, out = pipeline.execute() + self.assertEqual(0, exitcode, msg=out) + + message = project1.projectmessages.get() + self.assertEqual("find_origins", message.model) + self.assertIn("No such file or directory", message.description) + @mock.patch("scanpipe.pipes.scanoss.get_uvx_location") @mock.patch("scanpipe.pipes.scanoss.is_configured") @mock.patch("scanpipe.pipes.matchcode.is_configured")