From 51a270eaca9cd2a3f7651cffb6b81a666492b659 Mon Sep 17 00:00:00 2001 From: ziad hany Date: Tue, 22 Sep 2026 14:47:23 +0300 Subject: [PATCH 1/5] Create basic ScanCode.io pipeline and pipes for other reachability tools Signed-off-by: ziadhany --- docs/index.rst | 2 + ...utorial_reachability_integration_tools.rst | 236 ++++++++++++++++++ pyproject.toml | 1 + .../pipelines/analyze_symbols_reachability.py | 59 +---- scanpipe/pipes/reachability.py | 6 +- scanpipe/pipes/reachability_tools.py | 127 ++++++++++ 6 files changed, 378 insertions(+), 53 deletions(-) create mode 100644 docs/tutorial_reachability_integration_tools.rst create mode 100644 scanpipe/pipes/reachability_tools.py diff --git a/docs/index.rst b/docs/index.rst index 44e91c8abc..234542e246 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -48,6 +48,7 @@ Learn via practical step-by-step guides. - :ref:`tutorial_web_ui_symbol_and_string_collection` - :ref:`tutorial_cli_end_to_end_scanning_to_dejacode` - :ref:`tutorial_analyze_symbols_reachability` +- :ref:`tutorial_reachability_integration_tools` - :ref:`tutorial_web_ui_scan_repo_health` .. rst-class:: column column2 bottom-left @@ -119,6 +120,7 @@ Indices and tables tutorial_web_ui_symbol_and_string_collection tutorial_cli_end_to_end_scanning_to_dejacode tutorial_analyze_symbols_reachability + tutorial_reachability_integration_tools tutorial_web_ui_scan_repo_health scanpipe-concepts built-in-pipelines diff --git a/docs/tutorial_reachability_integration_tools.rst b/docs/tutorial_reachability_integration_tools.rst new file mode 100644 index 0000000000..bea5184353 --- /dev/null +++ b/docs/tutorial_reachability_integration_tools.rst @@ -0,0 +1,236 @@ +.. _tutorial_reachability_integration_tools: + +Integrate Other Reachability Tools +================================== + +This tutorial shows how to integrate any reachability analysis tool with +ScanCode.io. + +Most open source code analysis tools can create a call graph and run query manually +based on that to get reachability details, but do not +provide a complete end-to-end workflow for deciding whether those +vulnerabilities are actually *reachable* from the application code. ScanCode.io +fills this gap by letting you plug in your own reachability analysis: + +1. Scan the project with the ``scan_codebase`` pipeline to collect its package + and file details. +2. Run the ``find_vulnerabilities`` pipeline to collect the list of all the + vulnerabilities found in the codebase. This requires a configured + VulnerableCode service. see: :ref:`tutorial_vulnerablecode_integration`. +3. Run your own reachability pipeline, such as ``awesome_reachability``, to + get a reachability report for each vulnerability. + + +How the integration works +-------------------------- + +The integration is based on two base classes, available in the +``scanpipe.pipes.reachability_tools`` module: + +- ``ReachabilityTool`` wraps a reachability analysis tool. It declares the tool + ``tool_name``, its ``executable``, and the programming languages it supports + (``supported_languages``), and implements the tool-specific logic: checking + the tool availability, running the tool, parsing its output. + +- ``ReachabilityPipeline`` orchestrates the whole workflow. + +The base pipeline already implements most of the steps: + +- ``get_candidate_resources``: Collect non-binary, non-archive, non-media source + files for analysis. + +- ``get_vulnerabilities_patches``: collects the vulnerability patches for the + project packages. + +- ``collect_patch_symbols``: collects the vulnerable and fixed symbols for each + patch. +- ``generate_advisory_reachability_report``: generates a reachability report for + each vulnerability advisory. +- ``apply_reachability_to_packages_and_dependencies``: stores the reachability + results on the matching packages and dependencies. + +What you need to implement +--------------------------- + +To integrate a reachability tool, you only need to: + +1. Subclass ``ReachabilityTool`` and implement the tool-specific logic, such + as: + + - ``get_availability``: return an error message when the tool is not + available, for example when its executable is not configured. + - ``run``: run the tool on the project codebase and return the path + to its output file. + - ``parsed_output``: load and return the tool output. The output format is + tool-specific; in this example, a JSON file. + +2. Subclass ``ReachabilityPipeline``, set its ``reachability_tool`` attribute + to your tool class, and implement the two remaining steps: + + - ``collect_resource_index``: run the tool on the project codebase and load + the resulting index of resource symbols, for example as + ``self.resource_indexes``. This is also where you select the resources + eligible for analysis, such as ``self.candidate_resources``. + - ``collect_and_match_resources``: match the resource symbols against the + vulnerable and fixed symbols of each patch, and record the reachability + results. This method can rely on the ``self.patches`` and + ``self.patch_symbols`` values collected by the base pipeline steps. + +Example integration +-------------------- + +The following example integrates a fictional ``awesome_reachability`` tool. +Replace the tool name, executable, command line, output format, and matching +symbol logic with those of the tool you are integrating. + +.. code-block:: python + + AWESOME_TOOL_EXECUTABLE = environ.get("AWESOME_TOOL_EXECUTABLE") + + class AwesomeReachabilityTool(ReachabilityTool): + """Reachability analysis using the "awesome_reachability" tool.""" + + tool_name = "awesome_reachability" + executable = AWESOME_TOOL_EXECUTABLE + supported_language = ("Python", ...) + + @classmethod + def get_availability(cls): + """Return an error message when the tool is not available.""" + if not cls.executable: + return "The awesome_reachability tool is not configured." + + @classmethod + def run(cls, project, logger=None): + ... + + @classmethod + def parsed_output(cls, target_path): + ... + + + class AwesomeReachability(ReachabilityPipeline): + reachability_tool = AwesomeReachabilityTool + + def collect_resource_index(self): + ... + + def collect_and_match_resources(self): + ... + + +Data structures +~~~~~~~~~~~~~~~ + +The base pipeline collects the vulnerability data and exposes it as +pipeline attributes. When implementing ``match_patches_to_resources`` +and ``collect_and_match_resources``, you can rely on the following +structures. + +``patches`` +++++++++++++++++ + +Collected by the ``get_vulnerabilities_patches`` step, this is a list of +unique patches. Each patch is identified by a ``(vcs_url, commit_hash)`` +pair and lists the identifiers of the advisories it fixes: + +.. code-block:: python + + self.patches = [ + { + "vcs_url": "https://github.com/aboutcode-org/test", + "commit_hash": "07ec0de1964b14bf085a1c9a27ece2b61ab6105c", + "advisory_uids": ["PYSEC-2025-1"], + }, + ] + +``patch_symbols`` +++++++++++++++++++++++ + +Collected by the ``collect_patch_symbols`` step, this maps each +``commit_hash`` to the vulnerable and fixed symbols extracted from the +patch, grouped by programming language: + +.. code-block:: python + + self.patch_symbols = { + "07ec0de1964b14bf085a1c9a27ece2b61ab6105c": { + "Python": { + "vulnerable": { + "app.py::serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": "def build_file_path(filename): ...", + "fingerprint": "762e4f7d03b1bf4359c3ca364e55814...", + "start_line": 19, + "end_line": 22, + "node_type": "function_definition", + }, + }, + "fixed": { + "app.py::serve_report.build_file_path": { + "qualified_name": "serve_report.build_file_path", + "text": "def build_file_path(filename): ...", + "fingerprint": "646743b5d5497f6ea3b96f860bcbeb38...", + "start_line": 19, + "end_line": 25, + "node_type": "function_definition", + }, + }, + }, + }, + } + +Notes: + +- Symbols are keyed as ``"::"``, and + ``qualified_name`` uses the language separator (``.``) for nested + symbols, such as ``ClassName.method_name``. +- ``fingerprint`` is the SHA-256 of the symbol body ``text``. It is + used for exact matching between a patch symbol and a resource symbol. +- ``start_line`` and ``end_line`` are 1-based line numbers of the symbol + in its file. +- ``node_type`` is a syntax node type from tree-sitter, such as ``function_definition``, + ``class_definition``, .. +- The language keys, such as ``"Python"`` or ``"Java"``. + + +Run the ``awesome_reachability`` pipeline +------------------------------------------------- + +- Open any existing project containing a few resources. +- Click the **"Add pipeline"** button and select the **"awesome_reachability"** + pipeline from the dropdown list. +- Select **"Execute pipeline now"** and click **"Add pipeline"** to start the + reachability analysis. +- Once the pipeline run completes successfully, you can reach the **Resources** list view + by clicking the count number under the **"RESOURCES"** header. +- Click on one of the affected code files and navigate to + the **Extra** tab to view the ``symbols_reachability``. + +.. image:: images/tutorial-web-ui-analyze-symbol-reachability.png + +- The pipeline output must includes a JSON file containing the reachability + status for each advisory and resource, including the overall reachability + status (e.g., ``reachability-2026-08-18-15-12-51.json``). + +.. code-block:: json + :emphasize-lines: 2 + + { + "purl": "pkg:pypi/b2sdk@1.14.0", + "advisories": [ + { + "advisory_uid": "pypa/b2sdk/PYSEC-2022-33", + "is_reachable": "yes", + "details": [], + "vulnerable_symbols": [ + "SqliteAccountInfo", + "SqliteAccountInfo._create_database" + ], + "fixed_symbols": [ + "SqliteAccountInfo", + "SqliteAccountInfo._create_database" + ] + }] + } + } diff --git a/pyproject.toml b/pyproject.toml index c79e34f087..f261d1a7b1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,6 +155,7 @@ analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker" analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS" analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows" analyze_symbols_reachability = "scanpipe.pipelines.analyze_symbols_reachability:SymbolReachability" +cpg_symbols_reachability = "scanpipe.pipelines.cpg_symbols_reachability:CPGReachability" benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls" collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext" collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags" diff --git a/scanpipe/pipelines/analyze_symbols_reachability.py b/scanpipe/pipelines/analyze_symbols_reachability.py index 4eaf4116e1..98a11ed64e 100644 --- a/scanpipe/pipelines/analyze_symbols_reachability.py +++ b/scanpipe/pipelines/analyze_symbols_reachability.py @@ -20,12 +20,12 @@ # ScanCode.io is a free software code scanning tool from nexB Inc. and others. # Visit https://github.com/aboutcode-org/scancode.io for support and download. -from scanpipe.pipelines import Pipeline from scanpipe.pipes import reachability +from scanpipe.pipes.reachability_tools import ReachabilityPipeline from scanpipe.pipes.symbols import TS_QUERIES -class SymbolReachability(Pipeline): +class SymbolReachability(ReachabilityPipeline): """ Determine the reachability of vulnerabilities identified in the project. @@ -45,46 +45,21 @@ class SymbolReachability(Pipeline): advisory and saved as a JSON output file. """ - download_inputs = False - is_addon = True - results_url = "/project/{slug}/resources/?extra_data=symbol_reachability" - - @classmethod - def steps(cls): - return ( - cls.get_vulnerabilities_patches, - cls.collect_resource_index, - cls.collect_patch_symbols, - cls.collect_and_match_resources, - cls.generate_advisory_reachability_report, - cls.apply_reachability_to_packages_and_dependencies, - ) - - def get_vulnerabilities_patches(self): - """Get unique patch for all vulnerabilities.""" - self.patches = reachability.get_vulnerabilities_patches( - package_vulnerabilities=self.project.package_vulnerabilities, - dependency_vulnerabilities=self.project.dependency_vulnerabilities, - ) - - def collect_resource_index(self): - """Collect resources symbols for each resource""" + def get_candidate_resources(self): + """Collect candidate resources your pipeline needs.""" self.candidate_resources = self.project.codebaseresources.files().filter( is_binary=False, is_archive=False, is_media=False, programming_language__in=TS_QUERIES.keys(), ) + + def collect_resource_index(self): + """Collect resources symbols for each resource""" self.resource_indexes = reachability.collect_resource_index( candidate_resources=self.candidate_resources, logger=self.log ) - def collect_patch_symbols(self): - """Collect patch symbols for all related commits.""" - self.patch_symbols = reachability.collect_patch_symbols( - patches=self.patches, logger=self.log - ) - def collect_and_match_resources(self): """Match resource symbols against patch symbols.""" reachability.match_patches_to_resources( @@ -94,23 +69,3 @@ def collect_and_match_resources(self): candidate_resources=self.candidate_resources, logger=self.log, ) - - def generate_advisory_reachability_report(self): - """Generate a reachability report summarizing status by advisory.""" - self.advisories_reachability_report = ( - reachability.generate_advisory_reachability_report( - project=self.project, - patches=self.patches, - candidate_resources=self.candidate_resources, - ) - ) - - def apply_reachability_to_packages_and_dependencies(self): - """ - Save reachability results by updating DiscoveredPackage and - DiscoveredDependency records with the computed reachability data - in their affected_by_vulnerabilities JSON field. - """ - reachability.apply_reachability_to_packages_and_dependencies( - project=self.project, advisory_report=self.advisories_reachability_report - ) diff --git a/scanpipe/pipes/reachability.py b/scanpipe/pipes/reachability.py index ee0dc5993a..0c6423929d 100644 --- a/scanpipe/pipes/reachability.py +++ b/scanpipe/pipes/reachability.py @@ -614,7 +614,11 @@ def save_resource_reachability_report(resource, commit_hash, vcs_url, new_report old_commit_hash = patch_info.get("commit_hash") old_vcs_url = patch_info.get("vcs_url") - if old_commit_hash == commit_hash and old_vcs_url == vcs_url: + if ( + old_commit_hash == commit_hash + and old_vcs_url == vcs_url + and old_report.get("tool_name") == new_report.get("tool_name") + ): if not replaced: cleaned_reports.append(new_report) replaced = True diff --git a/scanpipe/pipes/reachability_tools.py b/scanpipe/pipes/reachability_tools.py new file mode 100644 index 0000000000..271039a81d --- /dev/null +++ b/scanpipe/pipes/reachability_tools.py @@ -0,0 +1,127 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +from abc import ABC + +from scanpipe.pipelines import Pipeline +from scanpipe.pipes import reachability + + +class ReachabilityTool(ABC): + """Base class for the analysis tools used by ReachabilityPipeline.""" + + tool_name = "" + executable = None + supported_languages = () + + @classmethod + def get_availability(cls): + return NotImplementedError + + @classmethod + def run(cls, project, logger=None): + raise NotImplementedError + + @classmethod + def parsed_output( + cls, + export_output_path, + logger=None, + ): + raise NotImplementedError + + +class ReachabilityPipeline(Pipeline): + """Base pipeline for the vulnerability reachability analysis""" + + download_inputs = False + is_addon = True + results_url = "/project/{slug}/resources/?extra_data=symbol_reachability" + reachability_tool = None + + @classmethod + def steps(cls): + return ( + cls.get_vulnerabilities_patches, + cls.get_candidate_resources, + cls.collect_resource_index, + cls.collect_patch_symbols, + cls.collect_and_match_resources, + cls.generate_advisory_reachability_report, + cls.apply_reachability_to_packages_and_dependencies, + ) + + @classmethod + def get_availability(cls): + if cls.reachability_tool: + return cls.reachability_tool.get_availability() + return None + + def get_vulnerabilities_patches(self): + """Get unique patch for all vulnerabilities.""" + self.patches = reachability.get_vulnerabilities_patches( + package_vulnerabilities=self.project.package_vulnerabilities, + dependency_vulnerabilities=self.project.dependency_vulnerabilities, + ) + + def get_candidate_resources(self): + """Collect candidate resources your pipeline needs.""" + self.candidate_resources = self.project.codebaseresources.files().filter( + is_binary=False, + is_archive=False, + is_media=False, + programming_language__in=self.reachability_tool.supported_languages, + ) + + def collect_resource_index(self): + """Collect the tool-specific resource index of the codebase.""" + raise NotImplementedError + + def collect_patch_symbols(self): + """Collect patch symbols for all related commits.""" + self.patch_symbols = reachability.collect_patch_symbols( + patches=self.patches, logger=self.log + ) + + def collect_and_match_resources(self): + """Match resource symbols against patch symbols.""" + raise NotImplementedError + + def generate_advisory_reachability_report(self): + """Generate a reachability report summarizing status by advisory.""" + self.advisories_reachability_report = ( + reachability.generate_advisory_reachability_report( + project=self.project, + patches=self.patches, + candidate_resources=self.candidate_resources, + ) + ) + + def apply_reachability_to_packages_and_dependencies(self): + """ + Save reachability results by updating DiscoveredPackage and + DiscoveredDependency records with the computed reachability data + in their affected_by_vulnerabilities JSON field. + """ + reachability.apply_reachability_to_packages_and_dependencies( + project=self.project, advisory_report=self.advisories_reachability_report + ) From 24e023b466cf47aaa7e66c2ff3f8a8de1cca801c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 7 Oct 2026 01:31:12 +0300 Subject: [PATCH 2/5] fix typo in pyproject.toml and improve reachability pipeline docs Signed-off-by: ziadhany --- pyproject.toml | 1 - scanpipe/pipes/reachability_tools.py | 18 ++++++++++++++++-- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index f261d1a7b1..c79e34f087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -155,7 +155,6 @@ analyze_docker_image = "scanpipe.pipelines.analyze_docker:Docker" analyze_root_filesystem_or_vm_image = "scanpipe.pipelines.analyze_root_filesystem:RootFS" analyze_windows_docker_image = "scanpipe.pipelines.analyze_docker_windows:DockerWindows" analyze_symbols_reachability = "scanpipe.pipelines.analyze_symbols_reachability:SymbolReachability" -cpg_symbols_reachability = "scanpipe.pipelines.cpg_symbols_reachability:CPGReachability" benchmark_purls = "scanpipe.pipelines.benchmark_purls:BenchmarkPurls" collect_strings_gettext = "scanpipe.pipelines.collect_strings_gettext:CollectStringsGettext" collect_symbols_ctags = "scanpipe.pipelines.collect_symbols_ctags:CollectSymbolsCtags" diff --git a/scanpipe/pipes/reachability_tools.py b/scanpipe/pipes/reachability_tools.py index 271039a81d..74e5e158f6 100644 --- a/scanpipe/pipes/reachability_tools.py +++ b/scanpipe/pipes/reachability_tools.py @@ -27,7 +27,11 @@ class ReachabilityTool(ABC): - """Base class for the analysis tools used by ReachabilityPipeline.""" + """ + Base class for the analysis tools used by ReachabilityPipeline. + Uses: + Subclasses must implement ``get_availability``, ``run``, and ``parsed_output``. + """ tool_name = "" executable = None @@ -35,10 +39,12 @@ class ReachabilityTool(ABC): @classmethod def get_availability(cls): + """Return None if this tool is available, or a reason string if not.""" return NotImplementedError @classmethod def run(cls, project, logger=None): + """Run the analysis tool.""" raise NotImplementedError @classmethod @@ -47,11 +53,18 @@ def parsed_output( export_output_path, logger=None, ): + """Parse and return the tool results from the output file.""" raise NotImplementedError class ReachabilityPipeline(Pipeline): - """Base pipeline for the vulnerability reachability analysis""" + """ + Base pipeline for the vulnerability reachability analysis + Uses: + Subclasses must implement ``collect_resource_index`` + and ``collect_and_match_resources`` and define + the ``reachability_tool`` attribute. + """ download_inputs = False is_addon = True @@ -72,6 +85,7 @@ def steps(cls): @classmethod def get_availability(cls): + """Return None if this pipeline is available, or a reason string if not.""" if cls.reachability_tool: return cls.reachability_tool.get_availability() return None From f855529533a3af2e1545520f2c2164a30348116c Mon Sep 17 00:00:00 2001 From: ziad hany Date: Wed, 7 Oct 2026 02:34:12 +0300 Subject: [PATCH 3/5] add a simple test for the reachability integration pipeline Signed-off-by: ziadhany --- .../tests/pipes/test_reachability_tools.py | 87 +++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 scanpipe/tests/pipes/test_reachability_tools.py diff --git a/scanpipe/tests/pipes/test_reachability_tools.py b/scanpipe/tests/pipes/test_reachability_tools.py new file mode 100644 index 0000000000..17ca4d6697 --- /dev/null +++ b/scanpipe/tests/pipes/test_reachability_tools.py @@ -0,0 +1,87 @@ +# SPDX-License-Identifier: Apache-2.0 +# +# http://nexb.com and https://github.com/aboutcode-org/scancode.io +# The ScanCode.io software is licensed under the Apache License version 2.0. +# Data generated with ScanCode.io is provided as-is without warranties. +# ScanCode is a trademark of nexB Inc. +# +# You may not use this software except in compliance with the License. +# You may obtain a copy of the License at: http://apache.org/licenses/LICENSE-2.0 +# Unless required by applicable law or agreed to in writing, software distributed +# under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +# CONDITIONS OF ANY KIND, either express or implied. See the License for the +# specific language governing permissions and limitations under the License. +# +# Data Generated with ScanCode.io is provided on an "AS IS" BASIS, WITHOUT WARRANTIES +# OR CONDITIONS OF ANY KIND, either express or implied. No content created from +# ScanCode.io should be considered or used as legal advice. Consult an Attorney +# for any legal advice. +# +# ScanCode.io is a free software code scanning tool from nexB Inc. and others. +# Visit https://github.com/aboutcode-org/scancode.io for support and download. + +import unittest + +from scanpipe.pipes.reachability_tools import ReachabilityPipeline +from scanpipe.pipes.reachability_tools import ReachabilityTool + + +class AwesomeReachabilityTool(ReachabilityTool): + tool_name = "awesome_reachability" + executable = "/awesome_reachability/tool" + supported_languages = ("Python",) + + @classmethod + def get_availability(cls): + if not cls.executable: + return "The awesome_reachability tool is not configured." + + @classmethod + def run(cls, project, logger=None): + return "awesome-output.json" + + @classmethod + def parsed_output(cls, export_output_path, logger=None): + return [{"symbol": "symbol_details"}] + + +class AwesomeReachability(ReachabilityPipeline): + reachability_tool = AwesomeReachabilityTool + + def collect_resource_index(self): + self.resource_indexes = {} + + def collect_and_match_resources(self): + return {} + + +class ReachabilityToolsTest(unittest.TestCase): + def test_tool_run(self): + self.assertEqual( + "awesome-output.json", AwesomeReachabilityTool.run(project=None) + ) + + def test_tool_parsed_output(self): + self.assertEqual( + [{"symbol": "symbol_details"}], + AwesomeReachabilityTool.parsed_output(export_output_path="output.json"), + ) + + def test_tool_get_availability(self): + self.assertIsNone(AwesomeReachabilityTool.get_availability()) + + class UnavailableTool(AwesomeReachabilityTool): + executable = None + + self.assertEqual( + "The awesome_reachability tool is not configured.", + UnavailableTool.get_availability(), + ) + + def test_pipeline_get_availability(self): + self.assertIsNone(AwesomeReachability.get_availability()) + + class NoToolPipeline(ReachabilityPipeline): + pass + + self.assertIsNone(NoToolPipeline.get_availability()) From a5329e3d202894f26fbb6e52f9deea6b7e82e64e Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 15 Oct 2026 03:41:15 +0300 Subject: [PATCH 4/5] Update reachability integration layer docs Signed-off-by: ziadhany --- ...utorial_reachability_integration_tools.rst | 4 +- scanpipe/pipes/reachability_tools.py | 59 +++++++++++++++++-- 2 files changed, 56 insertions(+), 7 deletions(-) diff --git a/docs/tutorial_reachability_integration_tools.rst b/docs/tutorial_reachability_integration_tools.rst index bea5184353..ab2143e4ea 100644 --- a/docs/tutorial_reachability_integration_tools.rst +++ b/docs/tutorial_reachability_integration_tools.rst @@ -231,6 +231,6 @@ Run the ``awesome_reachability`` pipeline "SqliteAccountInfo", "SqliteAccountInfo._create_database" ] - }] } - } + ] + } \ No newline at end of file diff --git a/scanpipe/pipes/reachability_tools.py b/scanpipe/pipes/reachability_tools.py index 74e5e158f6..b9fac9f27a 100644 --- a/scanpipe/pipes/reachability_tools.py +++ b/scanpipe/pipes/reachability_tools.py @@ -30,7 +30,9 @@ class ReachabilityTool(ABC): """ Base class for the analysis tools used by ReachabilityPipeline. Uses: - Subclasses must implement ``get_availability``, ``run``, and ``parsed_output``. + Subclasses must implement ``get_availability`` and ``run``. + They may optionally implement ``parsed_output`` to parse the + tool's output file, if applicable. """ tool_name = "" @@ -44,7 +46,12 @@ def get_availability(cls): @classmethod def run(cls, project, logger=None): - """Run the analysis tool.""" + """ + Run the analysis tool on the given project and return its output path + or store the results directly in an external system. + you can use this function in collect_resource_index + function to create the resuorce index graph + """ raise NotImplementedError @classmethod @@ -53,7 +60,12 @@ def parsed_output( export_output_path, logger=None, ): - """Parse and return the tool results from the output file.""" + """ + Parse and return the analysis results from the output file. + + Override this method if the tool generates an output file that + requires parsing, such as a JSON file. + """ raise NotImplementedError @@ -107,7 +119,18 @@ def get_candidate_resources(self): ) def collect_resource_index(self): - """Collect the tool-specific resource index of the codebase.""" + """ + Collect the resource index, the symbol index of the project resources, + possibly by running ``self.reachability_tool.run`` on the project. The + tool generates its analysis of the codebase, typically as a large JSON + output file, which is then parsed with + ``self.reachability_tool.parsed_output``; tools that store their + results directly skip the parsing. + + Return the resource index as a dict or JSON, or store it in + ``self.resource_indexes``. If the tool does not generate a + resource index, return None. + """ raise NotImplementedError def collect_patch_symbols(self): @@ -117,7 +140,33 @@ def collect_patch_symbols(self): ) def collect_and_match_resources(self): - """Match resource symbols against patch symbols.""" + """ + Match resource symbols against patch symbols, + and save one reachability report per matching resource. + + Each report is stored in the resource ``extra_data`` under + the ``symbols_reachability`` key, with the following structure:: + + { + "tool_name": "", + "patch": {"vcs_url": ..., "commit_hash": ...}, + "advisory_uids": [...], + "tool_details": [ + { + "symbol_name": ..., + "is_defined": ..., + ... + "reachable_from": [...], + }, + ], + "is_reachable": "yes" | "unknown" | "no", + } + + Implementations must save reports with + ``save_resource_reachability_report`` so that re-running the pipeline + replaces the previous report for the same (vcs_url, commit_hash, + tool_name) + """ raise NotImplementedError def generate_advisory_reachability_report(self): From afa7f784ec6c51071f89b63495312a362ea43dbe Mon Sep 17 00:00:00 2001 From: ziad hany Date: Thu, 15 Oct 2026 03:45:34 +0300 Subject: [PATCH 5/5] add the missing newline at end of tutorial_reachability_integration_tools file Signed-off-by: ziadhany --- docs/tutorial_reachability_integration_tools.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/tutorial_reachability_integration_tools.rst b/docs/tutorial_reachability_integration_tools.rst index ab2143e4ea..5ed43517ff 100644 --- a/docs/tutorial_reachability_integration_tools.rst +++ b/docs/tutorial_reachability_integration_tools.rst @@ -233,4 +233,4 @@ Run the ``awesome_reachability`` pipeline ] } ] - } \ No newline at end of file + }