Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions hamilton/caching/stores/memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -246,7 +246,8 @@ def load_from(

Since result stores do not store an index of their keys, you must provide a
``MetadataStore`` instance or a list of ``data_version`` for which results
should be loaded in memory.
should be loaded in memory. Missing results are skipped; stored ``None``
values are preserved.

:param result_store: ``ResultStore`` instance to load results from.
:param metadata_store: ``MetadataStore`` instance from which all ``data_version`` are retrieved.
Expand Down Expand Up @@ -289,9 +290,8 @@ def load_from(
data_versions_to_retrieve.add(node_metadata["data_version"])

for data_version in data_versions_to_retrieve:
# TODO disambiguate "result is None" from the sentinel value when `data_version`
# is not found in `result_store`.
result = result_store.get(data_version)
in_memory_result_store.set(data_version, result)
if result is not None or result_store.exists(data_version):
in_memory_result_store.set(data_version, result)

return in_memory_result_store
33 changes: 32 additions & 1 deletion tests/caching/result_store/test_memory.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,10 @@

import pytest

from hamilton.caching.cache_key import create_cache_key
from hamilton.caching.stores.base import StoredResult
from hamilton.caching.stores.file import FileResultStore
from hamilton.caching.stores.memory import InMemoryResultStore
from hamilton.caching.stores.memory import InMemoryMetadataStore, InMemoryResultStore

# `result_store` is imported but not directly used because it's
# a pytest fixture automatically provided to tests
Expand Down Expand Up @@ -146,3 +147,33 @@ def test_load_from_must_have_metadata_store_or_data_versions(tmp_path):
file_result_store = FileResultStore(tmp_path)
with pytest.raises(ValueError):
InMemoryResultStore.load_from(result_store=file_result_store)


@pytest.mark.parametrize("result_store", [FileResultStore, InMemoryResultStore], indirect=True)
@pytest.mark.parametrize("selection", ["versions", "metadata", "both"])
def test_load_from_preserves_missing_results(result_store, selection): # noqa: F811
values = {"present": 42, "none": None, "false": False, "zero": 0, "empty": ""}
for data_version, value in values.items():
result_store.set(data_version, value)

versions = [*values, "missing"]
metadata = InMemoryMetadataStore()
metadata.initialize("run")
for data_version in versions:
metadata.set(
cache_key=create_cache_key(data_version, "code", {}),
data_version=data_version,
run_id="run",
)

restored = InMemoryResultStore.load_from(
result_store,
data_versions=versions if selection != "metadata" else None,
metadata_store=metadata if selection != "versions" else None,
)

assert not restored.exists("missing")
assert restored.get("missing") is None
for data_version, value in values.items():
assert restored.exists(data_version)
assert restored.get(data_version) == value
32 changes: 32 additions & 0 deletions tests/caching/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,38 @@
from tests.resources.dynamic_parallelism import parallel_linear_basic, parallelism_with_caching


def test_restored_result_store_recomputes_missing_results(tmp_path):
calls = []

def answer() -> int:
calls.append(1)
return 42

module = ad_hoc_utils.create_temporary_module(answer)
metadata = InMemoryMetadataStore()
disk = FileResultStore(tmp_path)
first = (
driver.Builder()
.with_modules(module)
.with_cache(metadata_store=metadata, result_store=disk)
.build()
)
assert first.execute(["answer"]) == {"answer": 42}
disk.delete_all()

restored = InMemoryResultStore.load_from(disk, metadata_store=metadata)
second = (
driver.Builder()
.with_modules(module)
.with_cache(metadata_store=metadata, result_store=restored)
.build()
)
assert second.execute(["answer"]) == {"answer": 42}
assert len(calls) == 2
assert second.execute(["answer"]) == {"answer": 42}
assert len(calls) == 2


def _instantiate_executor(executor_cls):
if executor_cls == SynchronousLocalTaskExecutor:
return SynchronousLocalTaskExecutor()
Expand Down