diff --git a/dbsync.py b/dbsync.py index 143f62f..8f54694 100644 --- a/dbsync.py +++ b/dbsync.py @@ -7,6 +7,7 @@ """ import getpass +import glob import json import os import shutil @@ -557,6 +558,11 @@ def _get_db_project_comment(conn, schema) -> typing.Optional[typing.Dict]: return comment +def _download_project(mc, conn_cfg, work_dir, version=None): + """Download only the sync file of the Mergin Maps project (sparse checkout)""" + mc.download_project(conn_cfg.mergin_project, work_dir, version, include=[glob.escape(conn_cfg.sync_file)]) + + def _redownload_project(conn_cfg, mc, work_dir, db_proj_info): logging.debug(f"Removing local working directory {work_dir}") shutil.rmtree(work_dir) @@ -565,11 +571,7 @@ def _redownload_project(conn_cfg, mc, work_dir, db_proj_info): f"to {work_dir}" ) try: - mc.download_project( - conn_cfg.mergin_project, - work_dir, - db_proj_info["version"], - ) + _download_project(mc, conn_cfg, work_dir, db_proj_info["version"]) except ClientError as e: raise DbSyncError("Mergin Maps client error: " + str(e)) @@ -890,7 +892,11 @@ def status(conn_cfg, mc): server_info, ) - status_push = mp.get_push_changes() + try: + status_pull, status_push, _ = mc.project_status(work_dir) + except ClientError as e: + raise DbSyncError("Mergin Maps client error: " + str(e)) + if status_push["added"] or status_push["updated"] or status_push["removed"]: raise DbSyncError("Pending changes in the local directory - that should never happen! " + str(status_push)) @@ -899,7 +905,6 @@ def status(conn_cfg, mc): logging.debug("") logging.debug("Server is at version " + server_info["version"]) - status_pull = mp.get_pull_changes(server_info["files"], server_info["version"]) if status_pull["added"] or status_pull["updated"] or status_pull["removed"]: logging.debug("There are pending changes on server:") _print_mergin_changes(status_pull) @@ -1147,7 +1152,7 @@ def init( f"You either need to remove schema `{conn_cfg.base}` from Database or use `--force-init` option. " f"{FORCE_INIT_MESSAGE}" ) - mc.download_project(conn_cfg.mergin_project, work_dir, db_proj_info["version"]) + _download_project(mc, conn_cfg, work_dir, db_proj_info["version"]) else: # Get project ID from DB if available try: @@ -1173,7 +1178,7 @@ def init( else: if not os.path.exists(work_dir): logging.debug("Downloading latest Mergin Maps project " + conn_cfg.mergin_project + " to " + work_dir) - mc.download_project(conn_cfg.mergin_project, work_dir) + _download_project(mc, conn_cfg, work_dir) else: local_version = _get_project_version(work_dir) logging.debug(f"Working directory {work_dir} already exists, with project version {local_version}") @@ -1493,10 +1498,7 @@ def clean(conn_cfg, mc): try: # to remove sync file, download project to created directory, drop file and push changes back file = temp_folder / conn_cfg.sync_file - mc.download_project( - conn_cfg.mergin_project, - str(temp_folder), - ) + _download_project(mc, conn_cfg, str(temp_folder)) if file.exists(): file.unlink() mc.push_project(str(temp_folder)) diff --git a/requirements.txt b/requirements.txt index 4fe3f08..69cdfd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -mergin-client==0.13.3 +mergin-client==0.15.0 dynaconf>=3.1 psycopg2>=2.9 diff --git a/test/test_basic.py b/test/test_basic.py index d4497e4..31fab60 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -1,3 +1,4 @@ +import logging import uuid import pytest @@ -44,6 +45,7 @@ USER_PWD, SERVER_URL, TEST_DATA_DIR, + complete_project_name, init_sync_from_geopackage, ) @@ -558,9 +560,14 @@ def test_with_local_changes( project_name, source_gpkg_path, [], + None, *extra_files, ) + # sparse checkout - files other than the sync file are not downloaded + assert not os.path.exists(os.path.join(dbsync_project_dir, "note_1.txt")) + assert not os.path.exists(os.path.join(dbsync_project_dir, "modified_all.gpkg")) + # update GPKG shutil.copy( os.path.join( @@ -572,58 +579,24 @@ def test_with_local_changes( "test_sync.gpkg", ), ) - # update non-GPGK file - shutil.copy( - os.path.join( - TEST_DATA_DIR, - "note_2.txt", - ), - os.path.join( - dbsync_project_dir, - "note_1.txt", - ), - ) - # add GPKG file - shutil.copy( - os.path.join( - TEST_DATA_DIR, - "inserted_1_A.gpkg", - ), - os.path.join( - dbsync_project_dir, - "inserted_1_A.gpkg", - ), - ) - # add non-GPGK file - shutil.copy( - os.path.join( - TEST_DATA_DIR, - "note_2.txt", - ), - os.path.join( - dbsync_project_dir, - "note_2.txt", - ), - ) - # remove GPKG file - os.remove( - os.path.join( - dbsync_project_dir, - "modified_all.gpkg", - ) - ) - # remove non-GPKG file - os.remove( - os.path.join( - dbsync_project_dir, - "note_3.txt", + # add files outside of the sparse checkout - these are ignored + for f in ["note_1.txt", "note_2.txt", "inserted_1_A.gpkg"]: + shutil.copy( + os.path.join( + TEST_DATA_DIR, + "note_2.txt" if f.endswith(".txt") else f, + ), + os.path.join( + dbsync_project_dir, + f, + ), ) - ) # Check local changes in the sync project dir mp = _get_mergin_project(dbsync_project_dir) local_changes = mp.get_push_changes() - del local_changes["renamed"] # Not supported anymore - assert all(local_changes.values()) is True + assert not local_changes["added"] + assert not local_changes["removed"] + assert [f["path"] for f in local_changes["updated"]] == ["test_sync.gpkg"] dbsync_pull(mc) local_changes = mp.get_push_changes() assert any(local_changes.values()) is False @@ -1050,3 +1023,50 @@ def test_init_with_include_tables( assert cur.fetchone()[0] == False cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 4 + + +def test_sync_ignores_other_project_files(mc: MerginClient, caplog): + """Changes of other files in the project (outside of the sparse checkout) do not affect sync""" + project_name = "test_sync_ignores_other_files" + db_schema_main = project_name + "_main" + source_gpkg_path = os.path.join(TEST_DATA_DIR, "base.gpkg") + project_dir = os.path.join(TMP_DIR, project_name + "_work") + dbsync_project_dir = name_project_checkout_dir(project_name) + + init_sync_from_geopackage( + mc, + project_name, + source_gpkg_path, + [], + None, + os.path.join(TEST_DATA_DIR, "note_1.txt"), + ) + + # add another file to the project on the server + shutil.copy(os.path.join(TEST_DATA_DIR, "note_2.txt"), os.path.join(project_dir, "note_2.txt")) + mc.push_project(project_dir) + + # status does not report it as pending server change + caplog.set_level(logging.DEBUG) + dbsync_status(mc) + assert "No pending changes on server." in caplog.text + + # pull moves to the new version without downloading the file + dbsync_pull(mc) + assert _get_mergin_project(dbsync_project_dir).version() == "v2" + assert not os.path.exists(os.path.join(dbsync_project_dir, "note_2.txt")) + + # push from DB keeps the other files on the server + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + cur.execute( + sql.SQL("INSERT INTO {}.simple (name, rating) VALUES ('insert in postgres', 123)").format( + sql.Identifier(db_schema_main) + ) + ) + cur.execute("COMMIT") + dbsync_push(mc) + + server_info = mc.project_info(complete_project_name(project_name)) + assert server_info["version"] == "v3" + assert sorted(f["path"] for f in server_info["files"]) == ["note_1.txt", "note_2.txt", "test_sync.gpkg"]