diff --git a/pysus/api/client.py b/pysus/api/client.py index 982fbde9..bde2a859 100644 --- a/pysus/api/client.py +++ b/pysus/api/client.py @@ -627,8 +627,15 @@ def get_local_hierarchy(self): dataset = parts[-2] if len(parts) > 2 else "Other" has_group = getattr(r, "group", None) is not None + # _get_dest_path nests as + # downloads//// when the file + # has a group and downloads/// + # otherwise, so the dataset sits one level further in for a + # grouped file. The two cases were swapped, which put grouped + # files under their group name and ungrouped files under the + # client name. if path_obj.is_file() and len(parts) > 3: - dataset = parts[-2] if has_group else parts[-3] + dataset = parts[-3] if has_group else parts[-2] client_dict = hierarchy.setdefault(client, {}) ds_dict = client_dict.setdefault(dataset, {}) diff --git a/pysus/tests/api/test_client.py b/pysus/tests/api/test_client.py index 860015a7..ab22f7c8 100644 --- a/pysus/tests/api/test_client.py +++ b/pysus/tests/api/test_client.py @@ -1179,3 +1179,103 @@ async def test_aenter(self, test_db_path): assert client._ducklake is not None await client.__aexit__(None, None, None) + + +class TestGetLocalHierarchy: + """`_get_dest_path` nests grouped files one level deeper. + + downloads//// (has a group) + downloads/// (no group) + + so the dataset is `parts[-3]` in the first case and `parts[-2]` in + the second. The two were swapped. + """ + + async def _add(self, local, client_name="ftp", **kwargs): + local.parent.mkdir(parents=True, exist_ok=True) + local.write_text("x") + await self.client._update_state( + local_path=local, + remote_path=f"/remote/{local.name}", + client_name=client_name, + status=DownloadStatus.COMPLETED, + **kwargs, + ) + + @pytest.mark.asyncio + async def test_grouped_file_is_keyed_by_dataset( + self, test_db_path, tmp_path + ): + self.client = PySUS(db_path=test_db_path) + local = tmp_path / "downloads" / "ftp" / "sinan" / "DC" / "DNAC2024.dbc" + await self._add(local, group="DC", year=2024) + + hierarchy = self.client.get_local_hierarchy() + + assert list(hierarchy) == ["FTP"] + assert list(hierarchy["FTP"]) == ["sinan"] + assert list(hierarchy["FTP"]["sinan"]) == ["DC"] + assert [f["name"] for f in hierarchy["FTP"]["sinan"]["DC"]] == [ + "DNAC2024.dbc" + ] + + await self.client.__aexit__(None, None, None) + + @pytest.mark.asyncio + async def test_ungrouped_file_is_keyed_by_dataset( + self, test_db_path, tmp_path + ): + self.client = PySUS(db_path=test_db_path) + local = tmp_path / "downloads" / "ftp" / "sim" / "SIM2024.csv" + await self._add(local) + + hierarchy = self.client.get_local_hierarchy() + + assert list(hierarchy) == ["FTP"] + assert list(hierarchy["FTP"]) == ["sim"] + assert list(hierarchy["FTP"]["sim"]) == [""] + assert [f["name"] for f in hierarchy["FTP"]["sim"][""]] == [ + "SIM2024.csv" + ] + + await self.client.__aexit__(None, None, None) + + @pytest.mark.asyncio + async def test_both_shapes_share_the_dataset(self, test_db_path, tmp_path): + self.client = PySUS(db_path=test_db_path) + await self._add( + tmp_path / "downloads" / "ftp" / "sinan" / "DC" / "DNAC2024.dbc", + group="DC", + ) + await self._add( + tmp_path / "downloads" / "ftp" / "sinan" / "SIH2024.dbf", + ) + + hierarchy = self.client.get_local_hierarchy() + + assert list(hierarchy["FTP"]) == ["sinan"] + assert sorted(hierarchy["FTP"]["sinan"]) == ["", "DC"] + + await self.client.__aexit__(None, None, None) + + @pytest.mark.asyncio + async def test_client_name_is_not_used_as_dataset( + self, test_db_path, tmp_path + ): + self.client = PySUS(db_path=test_db_path) + await self._add( + tmp_path / "downloads" / "ftp" / "sinan" / "DC" / "DNAC2024.dbc", + group="DC", + ) + await self._add( + tmp_path / "downloads" / "dadosgov" / "pni" / "PNI.csv", + client_name="dadosgov", + ) + + hierarchy = self.client.get_local_hierarchy() + + assert set(hierarchy) == {"FTP", "DADOSGOV"} + assert list(hierarchy["FTP"]) == ["sinan"] + assert list(hierarchy["DADOSGOV"]) == ["pni"] + + await self.client.__aexit__(None, None, None)