[code-genetics] Prototype: origin matches from MatchCode and SCANOSS

docs/application-settings.rst

@@ -381,6 +381,31 @@ API key using ``MATCHCODEIO_API_KEY``::
 
     MATCHCODEIO_API_KEY=insert_your_api_key_here
 
+.. _scancodeio_settings_scanoss:
+
+SCANOSS
+^^^^^^^
+
+`SCANOSS <https://www.scanoss.com/>`_ identifies the origin of files and code
+snippets. You can use the free public service, a premium service, or your own
+instance.
+
+To configure your local environment, set the ``SCANOSS_URL`` in your ``.env``
+file::
+
+    SCANOSS_URL=https://api.osskb.org
+
+If your SCANOSS service requires authentication, you can provide the API key
+using ``SCANOSS_API_KEY``::
+
+    SCANOSS_API_KEY=insert_your_api_key_here
+
+.. note::
+    The SCANOSS client is installed in a temporary environment for each scan,
+    using the ``uvx`` executable, which requires access to PyPI.
+    Only the file paths, checksums, sizes, and fingerprints are sent to the
+    SCANOSS service, not the file contents.
+
 .. _scancodeio_settings_federatedcode:
 
 FEDERATEDCODE

docs/built-in-pipelines.rst

@@ -140,6 +140,20 @@ Enrich With PurlDB (addon)
     :members:
     :member-order: bysource
 
+.. _pipeline_find_origins:
+
+Find Origins (addon)
+--------------------
+.. warning::
+    This pipeline requires access to at least one origin detection tool.
+    Refer to :ref:`scancodeio_settings_matchcodeio` and
+    :ref:`scancodeio_settings_scanoss` to configure access to MatchCode.io and
+    SCANOSS in your ScanCode.io instance.
+
+.. autoclass:: scanpipe.pipelines.find_origins.FindOrigins()
+    :members:
+    :member-order: bysource
+
 .. _pipeline_find_vulnerabilities:
 
 Find Vulnerabilities (addon)

pyproject.toml

@@ -162,6 +162,7 @@ collect_symbols_pygments = "scanpipe.pipelines.collect_symbols_pygments:CollectS
 collect_symbols_tree_sitter = "scanpipe.pipelines.collect_symbols_tree_sitter:CollectSymbolsTreeSitter"
 enrich_with_purldb = "scanpipe.pipelines.enrich_with_purldb:EnrichWithPurlDB"
 fetch_scores = "scanpipe.pipelines.fetch_scores:FetchScores"
+find_origins = "scanpipe.pipelines.find_origins:FindOrigins"
 find_vulnerabilities = "scanpipe.pipelines.find_vulnerabilities:FindVulnerabilities"
 inspect_elf_binaries = "scanpipe.pipelines.inspect_elf_binaries:InspectELFBinaries"
 inspect_packages = "scanpipe.pipelines.inspect_packages:InspectPackages"

scancodeio/settings.py

@@ -354,6 +354,11 @@
 MATCHCODEIO_PASSWORD = env.str("MATCHCODEIO_PASSWORD", default="")
 MATCHCODEIO_API_KEY = env.str("MATCHCODEIO_API_KEY", default="")
 
+# SCANOSS integration
+
+SCANOSS_URL = env.str("SCANOSS_URL", default="").rstrip("/")
+SCANOSS_API_KEY = env.str("SCANOSS_API_KEY", default="")
+
 # FederatedCode integration
 
 FEDERATEDCODE_GIT_ACCOUNT_URL = env.str(

scanpipe/admin.py

@@ -27,6 +27,7 @@
 from scanpipe.models import CodebaseResource
 from scanpipe.models import DiscoveredDependency
 from scanpipe.models import DiscoveredPackage
+from scanpipe.models import OriginMatch
 from scanpipe.models import Project
 
 
@@ -165,6 +166,32 @@ class DiscoveredDependencyAdmin(ScanPipeBaseAdmin):
     ordering = ["project", "dependency_uid"]
 
 
+class OriginMatchAdmin(ScanPipeBaseAdmin):
+    list_display = [
+        "purl",
+        "match_type",
+        "similarity",
+        "tool",
+        "resource",
+        "project",
+    ]
+    search_fields = [
+        "uuid",
+        "purl",
+        "origin_path",
+        "resource__path",
+    ]
+    list_filter = ["project", "match_type", "tool"]
+    list_select_related = ["resource", "project"]
+    ordering = ["project", "resource__path", "-similarity", "purl"]
+
+    def has_change_permission(self, request, obj=None):
+        return False
+
+    def has_delete_permission(self, request, obj=None):
+        return False
+
+
 class ScanCodeIOAdminSite(admin.AdminSite):
     site_header = "ScanCode.io administration"
     site_title = "ScanCode.io administration"
@@ -175,3 +202,4 @@ class ScanCodeIOAdminSite(admin.AdminSite):
 admin_site.register(CodebaseResource, CodebaseResourceAdmin)
 admin_site.register(DiscoveredPackage, DiscoveredPackageAdmin)
 admin_site.register(DiscoveredDependency, DiscoveredDependencyAdmin)
+admin_site.register(OriginMatch, OriginMatchAdmin)

scanpipe/migrations/0083_alter_discoveredlicense_review_comments_and_more.py

@@ -0,0 +1,49 @@
+# Generated by Django 6.1.2 on 2026-10-14 11:24
+
+import django.core.validators
+import django.db.models.deletion
+import scanpipe.models
+import uuid
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('scanpipe', '0082_project_labels_json'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='discoveredlicense',
+            name='review_comments',
+            field=models.JSONField(blank=True, default=list, help_text='A list of review comments for license detection issues which needs review. These descriptive comments are based on ambiguous detection types and could also offer helpful suggestions on how to review/report these detection issues.', verbose_name='Review Comments'),
+        ),
+        migrations.AlterField(
+            model_name='discoveredpackage',
+            name='datafile_paths',
+            field=models.JSONField(blank=True, default=list, help_text='A list of Resource paths for package datafiles which were used to assemble this package.'),
+        ),
+        migrations.CreateModel(
+            name='OriginMatch',
+            fields=[
+                ('uuid', models.UUIDField(db_index=True, default=uuid.uuid4, editable=False, primary_key=True, serialize=False, verbose_name='UUID')),
+                ('extra_data', models.JSONField(blank=True, default=dict, help_text='Optional mapping of extra data key/values.')),
+                ('purl', models.CharField(help_text='Package URL (PURL) of the candidate origin package.', max_length=2048)),
+                ('match_type', models.CharField(choices=[('package', 'Package'), ('file', 'File'), ('directory', 'Directory'), ('approximate', 'Approximate'), ('snippet', 'Snippet')], help_text='How the resource matched the origin package: the whole package archive, an exact file, a directory, an approximate file, or a snippet.', max_length=20)),
+                ('tool', models.CharField(help_text='Name of the tool that reported this match.', max_length=50)),
+                ('tool_version', models.CharField(blank=True, help_text='Version of the tool that reported this match.', max_length=100)),
+                ('origin_path', models.CharField(blank=True, help_text='Path of the matched file in the origin package.', max_length=2000)),
+                ('line_ranges', models.JSONField(blank=True, default=list, help_text="Matched line ranges, as a list of mappings with the 'lines' of the resource and the corresponding 'origin_lines' of the origin file.")),
+                ('similarity', models.PositiveSmallIntegerField(blank=True, help_text='Similarity between the resource and the origin file, from 0 to 100. Empty when the tool does not report it.', null=True, validators=[django.core.validators.MaxValueValidator(100)])),
+                ('project', models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name='%(class)ss', to='scanpipe.project')),
+                ('resource', models.ForeignKey(editable=False, help_text='The matched file or directory.', on_delete=django.db.models.deletion.CASCADE, related_name='origin_matches', to='scanpipe.codebaseresource')),
+            ],
+            options={
+                'ordering': ['resource__path', '-similarity', 'purl'],
+                'indexes': [models.Index(fields=['purl'], name='scanpipe_or_purl_ee2cf2_idx'), models.Index(fields=['match_type'], name='scanpipe_or_match_t_b45ad1_idx'), models.Index(fields=['tool'], name='scanpipe_or_tool_3437f2_idx'), models.Index(fields=['similarity'], name='scanpipe_or_similar_f8430f_idx')],
+                'constraints': [models.UniqueConstraint(fields=('resource', 'purl', 'match_type', 'tool', 'origin_path'), name='scanpipe_originmatch_unique_origin_match')],
+            },
+            bases=(scanpipe.models.UpdateMixin, models.Model),
+        ),
+    ]

scanpipe/models.py

@@ -42,6 +42,7 @@
 from django.core.exceptions import ValidationError
 from django.core.serializers.json import DjangoJSONEncoder
 from django.core.validators import EMPTY_VALUES
+from django.core.validators import MaxValueValidator
 from django.db import models
 from django.db import transaction
 from django.db.models import Case
@@ -705,6 +706,7 @@ def delete_related_objects(self, keep_input=False, keep_webhook=False):
             self.codebaserelations,
             self.discovereddependencies,
             self.discoveredlicenses,
+            self.originmatchs,
             self.codebaseresources,
             self.runs,
         ]
@@ -5047,3 +5049,97 @@ def create_from_data(cls, package_score, check):
             details=check.details or [],
             package_score=package_score,
         )
+
+
+class OriginMatchQuerySet(ProjectRelatedQuerySet):
+    def exact_matches(self):
+        return self.filter(match_type__in=self.model.EXACT_MATCH_TYPES)
+
+    def partial_matches(self):
+        return self.exclude(match_type__in=self.model.EXACT_MATCH_TYPES)
+
+
+class OriginMatch(UUIDPKModel, ProjectRelatedModel, ExtraDataFieldMixin):
+    """A candidate origin package for a resource, as reported by an origin tool."""
+
+    class MatchType(models.TextChoices):
+        PACKAGE = "package"
+        FILE = "file"
+        DIRECTORY = "directory"
+        APPROXIMATE = "approximate"
+        SNIPPET = "snippet"
+
+    # A directory match can be approximate, so only these types are exact.
+    EXACT_MATCH_TYPES = [MatchType.PACKAGE, MatchType.FILE]
+
+    resource = models.ForeignKey(
+        CodebaseResource,
+        related_name="origin_matches",
+        on_delete=models.CASCADE,
+        editable=False,
+        help_text=_("The matched file or directory."),
+    )
+    purl = models.CharField(
+        max_length=2048,
+        help_text=_("Package URL (PURL) of the candidate origin package."),
+    )
+    match_type = models.CharField(
+        max_length=20,
+        choices=MatchType.choices,
+        help_text=_(
+            "How the resource matched the origin package: the whole package "
+            "archive, an exact file, a directory, an approximate file, or a snippet."
+        ),
+    )
+    tool = models.CharField(
+        max_length=50,
+        help_text=_("Name of the tool that reported this match."),
+    )
+    tool_version = models.CharField(
+        max_length=100,
+        blank=True,
+        help_text=_("Version of the tool that reported this match."),
+    )
+    origin_path = models.CharField(
+        max_length=2000,
+        blank=True,
+        help_text=_("Path of the matched file in the origin package."),
+    )
+    line_ranges = models.JSONField(
+        default=list,
+        blank=True,
+        help_text=_(
+            "Matched line ranges, as a list of mappings with the 'lines' of the "
+            "resource and, when the tool reports them, the corresponding "
+            "'origin_lines' of the origin file."
+        ),
+    )
+    similarity = models.PositiveSmallIntegerField(
+        null=True,
+        blank=True,
+        validators=[MaxValueValidator(100)],
+        help_text=_(
+            "Similarity between the resource and the origin file, from 0 to 100. "
+            "Empty when the tool does not report it."
+        ),
+    )
+
+    objects = OriginMatchQuerySet.as_manager()
+
+    class Meta:
+        ordering = ["resource__path", "-similarity", "purl"]
+        indexes = [
+            models.Index(fields=["purl"]),
+            models.Index(fields=["match_type"]),
+            models.Index(fields=["tool"]),
+            models.Index(fields=["similarity"]),
+        ]
+        constraints = [
+            models.UniqueConstraint(
+                fields=["resource", "purl", "match_type", "tool", "origin_path"],
+                name="%(app_label)s_%(class)s_unique_origin_match",
+            ),
+        ]
+
+    def __str__(self):
+        return f"{self.purl} ({self.match_type} match by {self.tool})"

scanpipe/pipelines/find_origins.py

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: Apache-2.0
+#
+# http://nexb.com and https://github.com/aboutcode-org/scancode.io
+# The ScanCode.io software is licensed under the Apache License version 2.0.
+# Data generated with ScanCode.io is provided as-is without warranties.
+# ScanCode is a trademark of nexB Inc.
+#
+# You may not use this software except in compliance with the License.
+# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0
+# Unless required by applicable law or agreed to in writing, software distributed
+# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR
+# CONDITIONS OF ANY KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations under the License.
+#
+# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES
+# OR CONDITIONS OF ANY KIND, either express or implied. No content created from
+# ScanCode.io should be considered or used as legal advice. Consult an Attorney
+# for any legal advice.
+#
+# ScanCode.io is a free software code scanning tool from nexB Inc. and others.
+# Visit https://github.com/aboutcode-org/scancode.io for support and download.
+
+import subprocess
+
+from scanpipe.pipelines import Pipeline
+from scanpipe.pipes import matchcode
+from scanpipe.pipes import scanoss
+
+
+class FindOrigins(Pipeline):
+    """
+    Find the origin of the codebase files using origin detection tools.
+
+    The candidate origin packages reported by each configured tool are stored as
+    origin matches, to be combined and reviewed. A tool that is not configured is
+    skipped.
+
+    Supported tools:
+    - MatchCode.io: the codebase scan, including file and directory
+      fingerprints, is sent to the configured MatchCode.io server.
+    - SCANOSS: the codebase files are fingerprinted locally, then their paths,
+      checksums, sizes, and fingerprints are sent to the configured SCANOSS
+      server. The file contents are not sent.
+    """
+
+    download_inputs = False
+    is_addon = True
+
+    @classmethod
+    def steps(cls):
+        return (
+            cls.find_matchcode_origin_matches,
+            cls.find_scanoss_origin_matches,
+        )
+
+    @classmethod
+    def get_availability(cls):
+        if not matchcode.is_configured() and not scanoss.is_configured():
+            return "No origin detection tool is configured: MatchCode.io or SCANOSS."
+
+    def find_matchcode_origin_matches(self):
+        """Match the codebase with MatchCode.io and store the origin matches."""
+        if not matchcode.is_configured():
+            self.log("MatchCode.io is not configured, skipping.")
+            return
+
+        # A tool failure is recorded and does not prevent running the other tools.
+        with self.save_errors(matchcode.MatchCodeIOException):
+            match_results = matchcode.match_codebase(self.project)
+            matchcode.save_match_results(self.project, match_results)
+
+            origin_match_count = matchcode.load_matchcode_results(
+                self.project, match_results
+            )
+            self.log(f"{origin_match_count:,d} MatchCode.io origin matches processed.")
+
+    def find_scanoss_origin_matches(self):
+        """Scan the codebase with SCANOSS and store the reported origin matches."""
+        if not scanoss.is_configured():
+            self.log("SCANOSS is not configured, skipping.")
+            return
+
+        if not scanoss.get_uvx_location():
+            message = "The uvx executable required by SCANOSS is missing, skipping."
+            self.log(message)
+            self.project.add_warning(
+                description=message, model="find_scanoss_origin_matches"
+            )
+            return
+
+        results_location = self.project.get_output_file_path("scanoss", "json")
+        # A missing or invalid results file is recorded as a SCANOSS failure.
+        with self.save_errors(subprocess.SubprocessError, OSError, ValueError):
+            scanoss.scan_codebase(self.project.codebase_path, results_location)
+
+            origin_match_count = scanoss.load_scanoss_results(
+                self.project, results_location
+            )
+            self.log(f"{origin_match_count:,d} SCANOSS origin matches processed.")