Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion hamilton/caching/stores/sqlite.py
Original file line number Diff line number Diff line change
Expand Up @@ -163,9 +163,10 @@ def set(
cur.execute("INSERT INTO history (cache_key, run_id) VALUES (?, ?)", (cache_key, run_id))
cur.execute(
"""\
INSERT OR IGNORE INTO cache_metadata (
INSERT INTO cache_metadata (
cache_key, node_name, code_version, data_version
) VALUES (?, ?, ?, ?)
ON CONFLICT(cache_key) DO UPDATE SET data_version = excluded.data_version
""",
(cache_key, node_name, code_version, data_version),
)
Expand Down
18 changes: 18 additions & 0 deletions tests/caching/metadata_store/test_base.py
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,24 @@ def test_get_miss_returns_none(metadata_store):
assert data_version is None


@pytest.mark.parametrize("metadata_store", IMPLEMENTATIONS, indirect=True)
def test_set_updates_existing_data_version(metadata_store):
cache_key = _mock_cache_key()
for data_version in ["old", "fresh", "newest"]:
run_id = f"run-{data_version}"
metadata_store.initialize(run_id=run_id)
metadata_store.set(cache_key=cache_key, data_version=data_version, run_id=run_id)

assert metadata_store.get(cache_key) == data_version
assert metadata_store.size == 1
run_info = metadata_store.get_run(run_id)
assert len(run_info) == 1
assert run_info[0]["cache_key"] == cache_key
assert run_info[0]["data_version"] == data_version

assert metadata_store.get_run_ids() == ["run-old", "run-fresh", "run-newest"]


@pytest.mark.parametrize("metadata_store", IMPLEMENTATIONS, indirect=True)
def test_set_and_get_with_empty_dependencies(metadata_store):
cache_key = _mock_cache_key()
Expand Down
33 changes: 33 additions & 0 deletions tests/caching/test_integration.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
# under the License.

import itertools
from pathlib import Path

import pandas as pd
import pytest
Expand Down Expand Up @@ -439,6 +440,38 @@ def test_recompute_behavior(tmp_path):
check_results_exist_in_store(cache, ["A", "B"])


def test_recompute_refreshes_persistent_cache_for_later_default_runs(tmp_path):
def source_text(source_file: str) -> str:
return Path(source_file).read_text(encoding="utf-8")

def rendered(source_text: str) -> str:
return f"read:{source_text}"

module = ad_hoc_utils.create_temporary_module(source_text, rendered)
source_file = tmp_path / "source.txt"
cache_path = tmp_path / "cache"
inputs = {"source_file": str(source_file)}

source_file.write_text("old", encoding="utf-8")
initial = driver.Builder().with_modules(module).with_cache(path=cache_path).build()
assert initial.execute(["rendered"], inputs=inputs) == {"rendered": "read:old"}

source_file.write_text("fresh", encoding="utf-8")
refreshed = (
driver.Builder()
.with_modules(module)
.with_cache(path=cache_path, recompute=["source_text"])
.build()
)
assert refreshed.execute(["rendered"], inputs=inputs) == {"rendered": "read:fresh"}
check_execution(refreshed.cache, did=["source_text", "rendered"])

source_file.write_text("uncached", encoding="utf-8")
resumed = driver.Builder().with_modules(module).with_cache(path=cache_path).build()
assert resumed.execute(["rendered"], inputs=inputs) == {"rendered": "read:fresh"}
check_execution(resumed.cache, did_not=["source_text", "rendered"])


def test_disable_behavior(tmp_path):
cache = HamiltonCacheAdapter(path=tmp_path)
module = ad_hoc_utils.create_temporary_module(
Expand Down