diff --git a/hamilton/caching/stores/sqlite.py b/hamilton/caching/stores/sqlite.py index ec285e405..d23d2d199 100644 --- a/hamilton/caching/stores/sqlite.py +++ b/hamilton/caching/stores/sqlite.py @@ -163,9 +163,10 @@ def set( cur.execute("INSERT INTO history (cache_key, run_id) VALUES (?, ?)", (cache_key, run_id)) cur.execute( """\ - INSERT OR IGNORE INTO cache_metadata ( + INSERT INTO cache_metadata ( cache_key, node_name, code_version, data_version ) VALUES (?, ?, ?, ?) + ON CONFLICT(cache_key) DO UPDATE SET data_version = excluded.data_version """, (cache_key, node_name, code_version, data_version), ) diff --git a/tests/caching/metadata_store/test_base.py b/tests/caching/metadata_store/test_base.py index 79afd2868..f7fde14ff 100644 --- a/tests/caching/metadata_store/test_base.py +++ b/tests/caching/metadata_store/test_base.py @@ -123,6 +123,24 @@ def test_get_miss_returns_none(metadata_store): assert data_version is None +@pytest.mark.parametrize("metadata_store", IMPLEMENTATIONS, indirect=True) +def test_set_updates_existing_data_version(metadata_store): + cache_key = _mock_cache_key() + for data_version in ["old", "fresh", "newest"]: + run_id = f"run-{data_version}" + metadata_store.initialize(run_id=run_id) + metadata_store.set(cache_key=cache_key, data_version=data_version, run_id=run_id) + + assert metadata_store.get(cache_key) == data_version + assert metadata_store.size == 1 + run_info = metadata_store.get_run(run_id) + assert len(run_info) == 1 + assert run_info[0]["cache_key"] == cache_key + assert run_info[0]["data_version"] == data_version + + assert metadata_store.get_run_ids() == ["run-old", "run-fresh", "run-newest"] + + @pytest.mark.parametrize("metadata_store", IMPLEMENTATIONS, indirect=True) def test_set_and_get_with_empty_dependencies(metadata_store): cache_key = _mock_cache_key() diff --git a/tests/caching/test_integration.py b/tests/caching/test_integration.py index 93fdc06cf..eecccbdec 100644 --- a/tests/caching/test_integration.py +++ b/tests/caching/test_integration.py @@ -16,6 +16,7 @@ # under the License. import itertools +from pathlib import Path import pandas as pd import pytest @@ -439,6 +440,38 @@ def test_recompute_behavior(tmp_path): check_results_exist_in_store(cache, ["A", "B"]) +def test_recompute_refreshes_persistent_cache_for_later_default_runs(tmp_path): + def source_text(source_file: str) -> str: + return Path(source_file).read_text(encoding="utf-8") + + def rendered(source_text: str) -> str: + return f"read:{source_text}" + + module = ad_hoc_utils.create_temporary_module(source_text, rendered) + source_file = tmp_path / "source.txt" + cache_path = tmp_path / "cache" + inputs = {"source_file": str(source_file)} + + source_file.write_text("old", encoding="utf-8") + initial = driver.Builder().with_modules(module).with_cache(path=cache_path).build() + assert initial.execute(["rendered"], inputs=inputs) == {"rendered": "read:old"} + + source_file.write_text("fresh", encoding="utf-8") + refreshed = ( + driver.Builder() + .with_modules(module) + .with_cache(path=cache_path, recompute=["source_text"]) + .build() + ) + assert refreshed.execute(["rendered"], inputs=inputs) == {"rendered": "read:fresh"} + check_execution(refreshed.cache, did=["source_text", "rendered"]) + + source_file.write_text("uncached", encoding="utf-8") + resumed = driver.Builder().with_modules(module).with_cache(path=cache_path).build() + assert resumed.execute(["rendered"], inputs=inputs) == {"rendered": "read:fresh"} + check_execution(resumed.cache, did_not=["source_text", "rendered"]) + + def test_disable_behavior(tmp_path): cache = HamiltonCacheAdapter(path=tmp_path) module = ad_hoc_utils.create_temporary_module(