From cb467da497cd5e2e0212b6f89f8a482f8a82151d Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Wed, 7 Oct 2026 15:51:30 +0200 Subject: [PATCH 1/2] Download only the sync file from Mergin Maps project (#128) Use sparse checkout from mergin-client 0.15.0 so that db-sync downloads just the configured sync_file instead of the whole project (photos, basemaps etc.). Pull, push and status are limited to the sync file too. --- dbsync.py | 28 ++++++----- docs/using.md | 1 + requirements.txt | 2 +- test/test_basic.py | 116 ++++++++++++++++++++++++++------------------- 4 files changed, 85 insertions(+), 62 deletions(-) diff --git a/dbsync.py b/dbsync.py index f3a54fe..7e6179b 100644 --- a/dbsync.py +++ b/dbsync.py @@ -7,6 +7,7 @@ """ import getpass +import glob import json import os import shutil @@ -554,6 +555,11 @@ def _get_db_project_comment(conn, schema): return comment +def _download_project(mc, conn_cfg, work_dir, version=None): + """Download only the sync file of the Mergin Maps project (sparse checkout)""" + mc.download_project(conn_cfg.mergin_project, work_dir, version, include=[glob.escape(conn_cfg.sync_file)]) + + def _redownload_project(conn_cfg, mc, work_dir, db_proj_info): logging.debug(f"Removing local working directory {work_dir}") shutil.rmtree(work_dir) @@ -562,11 +568,7 @@ def _redownload_project(conn_cfg, mc, work_dir, db_proj_info): f"to {work_dir}" ) try: - mc.download_project( - conn_cfg.mergin_project, - work_dir, - db_proj_info["version"], - ) + _download_project(mc, conn_cfg, work_dir, db_proj_info["version"]) except ClientError as e: raise DbSyncError("Mergin Maps client error: " + str(e)) @@ -887,7 +889,11 @@ def status(conn_cfg, mc): server_info, ) - status_push = mp.get_push_changes() + try: + status_pull, status_push, _ = mc.project_status(work_dir) + except ClientError as e: + raise DbSyncError("Mergin Maps client error: " + str(e)) + if status_push["added"] or status_push["updated"] or status_push["removed"]: raise DbSyncError("Pending changes in the local directory - that should never happen! " + str(status_push)) @@ -896,7 +902,6 @@ def status(conn_cfg, mc): logging.debug("") logging.debug("Server is at version " + server_info["version"]) - status_pull = mp.get_pull_changes(server_info["files"], server_info["version"]) if status_pull["added"] or status_pull["updated"] or status_pull["removed"]: logging.debug("There are pending changes on server:") _print_mergin_changes(status_pull) @@ -1126,7 +1131,7 @@ def init( f"Downloading version {db_proj_info['version']} of Mergin Maps project {conn_cfg.mergin_project} " f"to {work_dir}" ) - mc.download_project(conn_cfg.mergin_project, work_dir, db_proj_info["version"]) + _download_project(mc, conn_cfg, work_dir, db_proj_info["version"]) else: # Get project ID from DB if available try: @@ -1156,7 +1161,7 @@ def init( else: if not os.path.exists(work_dir): logging.debug("Downloading latest Mergin Maps project " + conn_cfg.mergin_project + " to " + work_dir) - mc.download_project(conn_cfg.mergin_project, work_dir) + _download_project(mc, conn_cfg, work_dir) else: local_version = _get_project_version(work_dir) logging.debug(f"Working directory {work_dir} already exists, with project version {local_version}") @@ -1475,10 +1480,7 @@ def clean(conn_cfg, mc): try: # to remove sync file, download project to created directory, drop file and push changes back file = temp_folder / conn_cfg.sync_file - mc.download_project( - conn_cfg.mergin_project, - str(temp_folder), - ) + _download_project(mc, conn_cfg, str(temp_folder)) if file.exists(): file.unlink() mc.push_project(str(temp_folder)) diff --git a/docs/using.md b/docs/using.md index e91647d..eb7f701 100644 --- a/docs/using.md +++ b/docs/using.md @@ -59,6 +59,7 @@ connections: mergin_project: john/myproject # Path to the GeoPackage within the Mergin Maps project above # (it must exist if doing init from geopackage, it must not exist if doing init from database) + # Only this file is downloaded from the Mergin Maps project, other project files are ignored sync_file: data.gpkg daemon: diff --git a/requirements.txt b/requirements.txt index 4fe3f08..69cdfd4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -mergin-client==0.13.3 +mergin-client==0.15.0 dynaconf>=3.1 psycopg2>=2.9 diff --git a/test/test_basic.py b/test/test_basic.py index d4497e4..31fab60 100644 --- a/test/test_basic.py +++ b/test/test_basic.py @@ -1,3 +1,4 @@ +import logging import uuid import pytest @@ -44,6 +45,7 @@ USER_PWD, SERVER_URL, TEST_DATA_DIR, + complete_project_name, init_sync_from_geopackage, ) @@ -558,9 +560,14 @@ def test_with_local_changes( project_name, source_gpkg_path, [], + None, *extra_files, ) + # sparse checkout - files other than the sync file are not downloaded + assert not os.path.exists(os.path.join(dbsync_project_dir, "note_1.txt")) + assert not os.path.exists(os.path.join(dbsync_project_dir, "modified_all.gpkg")) + # update GPKG shutil.copy( os.path.join( @@ -572,58 +579,24 @@ def test_with_local_changes( "test_sync.gpkg", ), ) - # update non-GPGK file - shutil.copy( - os.path.join( - TEST_DATA_DIR, - "note_2.txt", - ), - os.path.join( - dbsync_project_dir, - "note_1.txt", - ), - ) - # add GPKG file - shutil.copy( - os.path.join( - TEST_DATA_DIR, - "inserted_1_A.gpkg", - ), - os.path.join( - dbsync_project_dir, - "inserted_1_A.gpkg", - ), - ) - # add non-GPGK file - shutil.copy( - os.path.join( - TEST_DATA_DIR, - "note_2.txt", - ), - os.path.join( - dbsync_project_dir, - "note_2.txt", - ), - ) - # remove GPKG file - os.remove( - os.path.join( - dbsync_project_dir, - "modified_all.gpkg", - ) - ) - # remove non-GPKG file - os.remove( - os.path.join( - dbsync_project_dir, - "note_3.txt", + # add files outside of the sparse checkout - these are ignored + for f in ["note_1.txt", "note_2.txt", "inserted_1_A.gpkg"]: + shutil.copy( + os.path.join( + TEST_DATA_DIR, + "note_2.txt" if f.endswith(".txt") else f, + ), + os.path.join( + dbsync_project_dir, + f, + ), ) - ) # Check local changes in the sync project dir mp = _get_mergin_project(dbsync_project_dir) local_changes = mp.get_push_changes() - del local_changes["renamed"] # Not supported anymore - assert all(local_changes.values()) is True + assert not local_changes["added"] + assert not local_changes["removed"] + assert [f["path"] for f in local_changes["updated"]] == ["test_sync.gpkg"] dbsync_pull(mc) local_changes = mp.get_push_changes() assert any(local_changes.values()) is False @@ -1050,3 +1023,50 @@ def test_init_with_include_tables( assert cur.fetchone()[0] == False cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main))) assert cur.fetchone()[0] == 4 + + +def test_sync_ignores_other_project_files(mc: MerginClient, caplog): + """Changes of other files in the project (outside of the sparse checkout) do not affect sync""" + project_name = "test_sync_ignores_other_files" + db_schema_main = project_name + "_main" + source_gpkg_path = os.path.join(TEST_DATA_DIR, "base.gpkg") + project_dir = os.path.join(TMP_DIR, project_name + "_work") + dbsync_project_dir = name_project_checkout_dir(project_name) + + init_sync_from_geopackage( + mc, + project_name, + source_gpkg_path, + [], + None, + os.path.join(TEST_DATA_DIR, "note_1.txt"), + ) + + # add another file to the project on the server + shutil.copy(os.path.join(TEST_DATA_DIR, "note_2.txt"), os.path.join(project_dir, "note_2.txt")) + mc.push_project(project_dir) + + # status does not report it as pending server change + caplog.set_level(logging.DEBUG) + dbsync_status(mc) + assert "No pending changes on server." in caplog.text + + # pull moves to the new version without downloading the file + dbsync_pull(mc) + assert _get_mergin_project(dbsync_project_dir).version() == "v2" + assert not os.path.exists(os.path.join(dbsync_project_dir, "note_2.txt")) + + # push from DB keeps the other files on the server + conn = psycopg2.connect(DB_CONNINFO) + cur = conn.cursor() + cur.execute( + sql.SQL("INSERT INTO {}.simple (name, rating) VALUES ('insert in postgres', 123)").format( + sql.Identifier(db_schema_main) + ) + ) + cur.execute("COMMIT") + dbsync_push(mc) + + server_info = mc.project_info(complete_project_name(project_name)) + assert server_info["version"] == "v3" + assert sorted(f["path"] for f in server_info["files"]) == ["note_1.txt", "note_2.txt", "test_sync.gpkg"] From 06e7ae2f0cc3f6e9cde91e9d9f1c6108308a36a3 Mon Sep 17 00:00:00 2001 From: Martin Varga Date: Thu, 8 Oct 2026 13:16:35 +0200 Subject: [PATCH 2/2] Remove comment --- docs/using.md | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/using.md b/docs/using.md index eb7f701..e91647d 100644 --- a/docs/using.md +++ b/docs/using.md @@ -59,7 +59,6 @@ connections: mergin_project: john/myproject # Path to the GeoPackage within the Mergin Maps project above # (it must exist if doing init from geopackage, it must not exist if doing init from database) - # Only this file is downloaded from the Mergin Maps project, other project files are ignored sync_file: data.gpkg daemon: