Download only the sync file from Mergin Maps project

dbsync.py

@@ -7,6 +7,7 @@
 """
 
 import getpass
+import glob
 import json
 import os
 import shutil
@@ -557,6 +558,11 @@ def _get_db_project_comment(conn, schema) -> typing.Optional[typing.Dict]:
     return comment
 
 
+def _download_project(mc, conn_cfg, work_dir, version=None):
+    """Download only the sync file of the Mergin Maps project (sparse checkout)"""
+    mc.download_project(conn_cfg.mergin_project, work_dir, version, include=[glob.escape(conn_cfg.sync_file)])
+
+
 def _redownload_project(conn_cfg, mc, work_dir, db_proj_info):
     logging.debug(f"Removing local working directory {work_dir}")
     shutil.rmtree(work_dir)
@@ -565,11 +571,7 @@ def _redownload_project(conn_cfg, mc, work_dir, db_proj_info):
         f"to {work_dir}"
     )
     try:
-        mc.download_project(
-            conn_cfg.mergin_project,
-            work_dir,
-            db_proj_info["version"],
-        )
+        _download_project(mc, conn_cfg, work_dir, db_proj_info["version"])
     except ClientError as e:
         raise DbSyncError("Mergin Maps client error: " + str(e))
 
@@ -890,7 +892,11 @@ def status(conn_cfg, mc):
         server_info,
     )
 
-    status_push = mp.get_push_changes()
+    try:
+        status_pull, status_push, _ = mc.project_status(work_dir)
+    except ClientError as e:
+        raise DbSyncError("Mergin Maps client error: " + str(e))
+
     if status_push["added"] or status_push["updated"] or status_push["removed"]:
         raise DbSyncError("Pending changes in the local directory - that should never happen! " + str(status_push))
 
@@ -899,7 +905,6 @@ def status(conn_cfg, mc):
     logging.debug("")
 
     logging.debug("Server is at version " + server_info["version"])
-    status_pull = mp.get_pull_changes(server_info["files"], server_info["version"])
     if status_pull["added"] or status_pull["updated"] or status_pull["removed"]:
         logging.debug("There are pending changes on server:")
         _print_mergin_changes(status_pull)
@@ -1147,7 +1152,7 @@ def init(
                         f"You either need to remove schema `{conn_cfg.base}` from Database or use `--force-init` option. "
                         f"{FORCE_INIT_MESSAGE}"
                     )
-            mc.download_project(conn_cfg.mergin_project, work_dir, db_proj_info["version"])
+            _download_project(mc, conn_cfg, work_dir, db_proj_info["version"])
         else:
             # Get project ID from DB if available
             try:
@@ -1173,7 +1178,7 @@ def init(
     else:
         if not os.path.exists(work_dir):
             logging.debug("Downloading latest Mergin Maps project " + conn_cfg.mergin_project + " to " + work_dir)
-            mc.download_project(conn_cfg.mergin_project, work_dir)
+            _download_project(mc, conn_cfg, work_dir)
         else:
             local_version = _get_project_version(work_dir)
             logging.debug(f"Working directory {work_dir} already exists, with project version {local_version}")
@@ -1493,10 +1498,7 @@ def clean(conn_cfg, mc):
         try:
             # to remove sync file, download project to created directory, drop file and push changes back
             file = temp_folder / conn_cfg.sync_file
-            mc.download_project(
-                conn_cfg.mergin_project,
-                str(temp_folder),
-            )
+            _download_project(mc, conn_cfg, str(temp_folder))
             if file.exists():
                 file.unlink()
             mc.push_project(str(temp_folder))

requirements.txt

@@ -1,3 +1,3 @@
-mergin-client==0.13.3
+mergin-client==0.15.0
 dynaconf>=3.1
 psycopg2>=2.9

test/test_basic.py

@@ -1,3 +1,4 @@
+import logging
 import uuid
 
 import pytest
@@ -44,6 +45,7 @@
     USER_PWD,
     SERVER_URL,
     TEST_DATA_DIR,
+    complete_project_name,
     init_sync_from_geopackage,
 )
 
@@ -558,9 +560,14 @@ def test_with_local_changes(
         project_name,
         source_gpkg_path,
         [],
+        None,
         *extra_files,
     )
 
+    # sparse checkout - files other than the sync file are not downloaded
+    assert not os.path.exists(os.path.join(dbsync_project_dir, "note_1.txt"))
+    assert not os.path.exists(os.path.join(dbsync_project_dir, "modified_all.gpkg"))
+
     # update GPKG
     shutil.copy(
         os.path.join(
@@ -572,58 +579,24 @@ def test_with_local_changes(
             "test_sync.gpkg",
         ),
     )
-    # update non-GPGK file
-    shutil.copy(
-        os.path.join(
-            TEST_DATA_DIR,
-            "note_2.txt",
-        ),
-        os.path.join(
-            dbsync_project_dir,
-            "note_1.txt",
-        ),
-    )
-    # add GPKG file
-    shutil.copy(
-        os.path.join(
-            TEST_DATA_DIR,
-            "inserted_1_A.gpkg",
-        ),
-        os.path.join(
-            dbsync_project_dir,
-            "inserted_1_A.gpkg",
-        ),
-    )
-    # add non-GPGK file
-    shutil.copy(
-        os.path.join(
-            TEST_DATA_DIR,
-            "note_2.txt",
-        ),
-        os.path.join(
-            dbsync_project_dir,
-            "note_2.txt",
-        ),
-    )
-    # remove GPKG file
-    os.remove(
-        os.path.join(
-            dbsync_project_dir,
-            "modified_all.gpkg",
-        )
-    )
-    # remove non-GPKG file
-    os.remove(
-        os.path.join(
-            dbsync_project_dir,
-            "note_3.txt",
+    # add files outside of the sparse checkout - these are ignored
+    for f in ["note_1.txt", "note_2.txt", "inserted_1_A.gpkg"]:
+        shutil.copy(
+            os.path.join(
+                TEST_DATA_DIR,
+                "note_2.txt" if f.endswith(".txt") else f,
+            ),
+            os.path.join(
+                dbsync_project_dir,
+                f,
+            ),
         )
-    )
     # Check local changes in the sync project dir
     mp = _get_mergin_project(dbsync_project_dir)
     local_changes = mp.get_push_changes()
-    del local_changes["renamed"]  # Not supported anymore
-    assert all(local_changes.values()) is True
+    assert not local_changes["added"]
+    assert not local_changes["removed"]
+    assert [f["path"] for f in local_changes["updated"]] == ["test_sync.gpkg"]
     dbsync_pull(mc)
     local_changes = mp.get_push_changes()
     assert any(local_changes.values()) is False
@@ -1050,3 +1023,50 @@ def test_init_with_include_tables(
     assert cur.fetchone()[0] == False
     cur.execute(sql.SQL("SELECT count(*) from {}.points").format(sql.Identifier(db_schema_main)))
     assert cur.fetchone()[0] == 4
+
+
+def test_sync_ignores_other_project_files(mc: MerginClient, caplog):
+    """Changes of other files in the project (outside of the sparse checkout) do not affect sync"""
+    project_name = "test_sync_ignores_other_files"
+    db_schema_main = project_name + "_main"
+    source_gpkg_path = os.path.join(TEST_DATA_DIR, "base.gpkg")
+    project_dir = os.path.join(TMP_DIR, project_name + "_work")
+    dbsync_project_dir = name_project_checkout_dir(project_name)
+
+    init_sync_from_geopackage(
+        mc,
+        project_name,
+        source_gpkg_path,
+        [],
+        None,
+        os.path.join(TEST_DATA_DIR, "note_1.txt"),
+    )
+
+    # add another file to the project on the server
+    shutil.copy(os.path.join(TEST_DATA_DIR, "note_2.txt"), os.path.join(project_dir, "note_2.txt"))
+    mc.push_project(project_dir)
+
+    # status does not report it as pending server change
+    caplog.set_level(logging.DEBUG)
+    dbsync_status(mc)
+    assert "No pending changes on server." in caplog.text
+
+    # pull moves to the new version without downloading the file
+    dbsync_pull(mc)
+    assert _get_mergin_project(dbsync_project_dir).version() == "v2"
+    assert not os.path.exists(os.path.join(dbsync_project_dir, "note_2.txt"))
+
+    # push from DB keeps the other files on the server
+    conn = psycopg2.connect(DB_CONNINFO)
+    cur = conn.cursor()
+    cur.execute(
+        sql.SQL("INSERT INTO {}.simple (name, rating) VALUES ('insert in postgres', 123)").format(
+            sql.Identifier(db_schema_main)
+        )
+    )
+    cur.execute("COMMIT")
+    dbsync_push(mc)
+
+    server_info = mc.project_info(complete_project_name(project_name))
+    assert server_info["version"] == "v3"
+    assert sorted(f["path"] for f in server_info["files"]) == ["note_1.txt", "note_2.txt", "test_sync.gpkg"]