From ebefd39d592ead81d5a8e78755855f9348205075 Mon Sep 17 00:00:00 2001 From: yangjiabin <706948683@qq.com> Date: Fri, 20 Nov 2026 22:08:30 +0800 Subject: [PATCH] fix(caching): preserve missing results when restoring in-memory caches --- hamilton/caching/stores/memory.py | 8 +++--- tests/caching/result_store/test_memory.py | 33 ++++++++++++++++++++++- tests/caching/test_integration.py | 32 ++++++++++++++++++++++ 3 files changed, 68 insertions(+), 5 deletions(-) diff --git a/hamilton/caching/stores/memory.py b/hamilton/caching/stores/memory.py index 7ed7ec49f..5d5bcdff4 100644 --- a/hamilton/caching/stores/memory.py +++ b/hamilton/caching/stores/memory.py @@ -246,7 +246,8 @@ def load_from( Since result stores do not store an index of their keys, you must provide a ``MetadataStore`` instance or a list of ``data_version`` for which results - should be loaded in memory. + should be loaded in memory. Missing results are skipped; stored ``None`` + values are preserved. :param result_store: ``ResultStore`` instance to load results from. :param metadata_store: ``MetadataStore`` instance from which all ``data_version`` are retrieved. @@ -289,9 +290,8 @@ def load_from( data_versions_to_retrieve.add(node_metadata["data_version"]) for data_version in data_versions_to_retrieve: - # TODO disambiguate "result is None" from the sentinel value when `data_version` - # is not found in `result_store`. result = result_store.get(data_version) - in_memory_result_store.set(data_version, result) + if result is not None or result_store.exists(data_version): + in_memory_result_store.set(data_version, result) return in_memory_result_store diff --git a/tests/caching/result_store/test_memory.py b/tests/caching/result_store/test_memory.py index b692393da..5d23d3611 100644 --- a/tests/caching/result_store/test_memory.py +++ b/tests/caching/result_store/test_memory.py @@ -17,9 +17,10 @@ import pytest +from hamilton.caching.cache_key import create_cache_key from hamilton.caching.stores.base import StoredResult from hamilton.caching.stores.file import FileResultStore -from hamilton.caching.stores.memory import InMemoryResultStore +from hamilton.caching.stores.memory import InMemoryMetadataStore, InMemoryResultStore # `result_store` is imported but not directly used because it's # a pytest fixture automatically provided to tests @@ -146,3 +147,33 @@ def test_load_from_must_have_metadata_store_or_data_versions(tmp_path): file_result_store = FileResultStore(tmp_path) with pytest.raises(ValueError): InMemoryResultStore.load_from(result_store=file_result_store) + + +@pytest.mark.parametrize("result_store", [FileResultStore, InMemoryResultStore], indirect=True) +@pytest.mark.parametrize("selection", ["versions", "metadata", "both"]) +def test_load_from_preserves_missing_results(result_store, selection): # noqa: F811 + values = {"present": 42, "none": None, "false": False, "zero": 0, "empty": ""} + for data_version, value in values.items(): + result_store.set(data_version, value) + + versions = [*values, "missing"] + metadata = InMemoryMetadataStore() + metadata.initialize("run") + for data_version in versions: + metadata.set( + cache_key=create_cache_key(data_version, "code", {}), + data_version=data_version, + run_id="run", + ) + + restored = InMemoryResultStore.load_from( + result_store, + data_versions=versions if selection != "metadata" else None, + metadata_store=metadata if selection != "versions" else None, + ) + + assert not restored.exists("missing") + assert restored.get("missing") is None + for data_version, value in values.items(): + assert restored.exists(data_version) + assert restored.get(data_version) == value diff --git a/tests/caching/test_integration.py b/tests/caching/test_integration.py index 93fdc06cf..5e4ddf15d 100644 --- a/tests/caching/test_integration.py +++ b/tests/caching/test_integration.py @@ -39,6 +39,38 @@ from tests.resources.dynamic_parallelism import parallel_linear_basic, parallelism_with_caching +def test_restored_result_store_recomputes_missing_results(tmp_path): + calls = [] + + def answer() -> int: + calls.append(1) + return 42 + + module = ad_hoc_utils.create_temporary_module(answer) + metadata = InMemoryMetadataStore() + disk = FileResultStore(tmp_path) + first = ( + driver.Builder() + .with_modules(module) + .with_cache(metadata_store=metadata, result_store=disk) + .build() + ) + assert first.execute(["answer"]) == {"answer": 42} + disk.delete_all() + + restored = InMemoryResultStore.load_from(disk, metadata_store=metadata) + second = ( + driver.Builder() + .with_modules(module) + .with_cache(metadata_store=metadata, result_store=restored) + .build() + ) + assert second.execute(["answer"]) == {"answer": 42} + assert len(calls) == 2 + assert second.execute(["answer"]) == {"answer": 42} + assert len(calls) == 2 + + def _instantiate_executor(executor_cls): if executor_cls == SynchronousLocalTaskExecutor: return SynchronousLocalTaskExecutor()