Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 8 additions & 1 deletion pysus/api/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -627,8 +627,15 @@ def get_local_hierarchy(self):
dataset = parts[-2] if len(parts) > 2 else "Other"
has_group = getattr(r, "group", None) is not None

# _get_dest_path nests as
# downloads/<client>/<dataset>/<group>/<name> when the file
# has a group and downloads/<client>/<dataset>/<name>
# otherwise, so the dataset sits one level further in for a
# grouped file. The two cases were swapped, which put grouped
# files under their group name and ungrouped files under the
# client name.
if path_obj.is_file() and len(parts) > 3:
dataset = parts[-2] if has_group else parts[-3]
dataset = parts[-3] if has_group else parts[-2]

client_dict = hierarchy.setdefault(client, {})
ds_dict = client_dict.setdefault(dataset, {})
Expand Down
100 changes: 100 additions & 0 deletions pysus/tests/api/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -1179,3 +1179,103 @@ async def test_aenter(self, test_db_path):
assert client._ducklake is not None

await client.__aexit__(None, None, None)


class TestGetLocalHierarchy:
"""`_get_dest_path` nests grouped files one level deeper.

downloads/<client>/<dataset>/<group>/<name> (has a group)
downloads/<client>/<dataset>/<name> (no group)

so the dataset is `parts[-3]` in the first case and `parts[-2]` in
the second. The two were swapped.
"""

async def _add(self, local, client_name="ftp", **kwargs):
local.parent.mkdir(parents=True, exist_ok=True)
local.write_text("x")
await self.client._update_state(
local_path=local,
remote_path=f"/remote/{local.name}",
client_name=client_name,
status=DownloadStatus.COMPLETED,
**kwargs,
)

@pytest.mark.asyncio
async def test_grouped_file_is_keyed_by_dataset(
self, test_db_path, tmp_path
):
self.client = PySUS(db_path=test_db_path)
local = tmp_path / "downloads" / "ftp" / "sinan" / "DC" / "DNAC2024.dbc"
await self._add(local, group="DC", year=2024)

hierarchy = self.client.get_local_hierarchy()

assert list(hierarchy) == ["FTP"]
assert list(hierarchy["FTP"]) == ["sinan"]
assert list(hierarchy["FTP"]["sinan"]) == ["DC"]
assert [f["name"] for f in hierarchy["FTP"]["sinan"]["DC"]] == [
"DNAC2024.dbc"
]

await self.client.__aexit__(None, None, None)

@pytest.mark.asyncio
async def test_ungrouped_file_is_keyed_by_dataset(
self, test_db_path, tmp_path
):
self.client = PySUS(db_path=test_db_path)
local = tmp_path / "downloads" / "ftp" / "sim" / "SIM2024.csv"
await self._add(local)

hierarchy = self.client.get_local_hierarchy()

assert list(hierarchy) == ["FTP"]
assert list(hierarchy["FTP"]) == ["sim"]
assert list(hierarchy["FTP"]["sim"]) == [""]
assert [f["name"] for f in hierarchy["FTP"]["sim"][""]] == [
"SIM2024.csv"
]

await self.client.__aexit__(None, None, None)

@pytest.mark.asyncio
async def test_both_shapes_share_the_dataset(self, test_db_path, tmp_path):
self.client = PySUS(db_path=test_db_path)
await self._add(
tmp_path / "downloads" / "ftp" / "sinan" / "DC" / "DNAC2024.dbc",
group="DC",
)
await self._add(
tmp_path / "downloads" / "ftp" / "sinan" / "SIH2024.dbf",
)

hierarchy = self.client.get_local_hierarchy()

assert list(hierarchy["FTP"]) == ["sinan"]
assert sorted(hierarchy["FTP"]["sinan"]) == ["", "DC"]

await self.client.__aexit__(None, None, None)

@pytest.mark.asyncio
async def test_client_name_is_not_used_as_dataset(
self, test_db_path, tmp_path
):
self.client = PySUS(db_path=test_db_path)
await self._add(
tmp_path / "downloads" / "ftp" / "sinan" / "DC" / "DNAC2024.dbc",
group="DC",
)
await self._add(
tmp_path / "downloads" / "dadosgov" / "pni" / "PNI.csv",
client_name="dadosgov",
)

hierarchy = self.client.get_local_hierarchy()

assert set(hierarchy) == {"FTP", "DADOSGOV"}
assert list(hierarchy["FTP"]) == ["sinan"]
assert list(hierarchy["DADOSGOV"]) == ["pni"]

await self.client.__aexit__(None, None, None)
Loading