From 308fce263f32261173dc7d4abd039b56e97d92eb Mon Sep 17 00:00:00 2001 From: mcarans Date: Tue, 20 Oct 2026 11:08:48 +1300 Subject: [PATCH] Add wrappers for HDX data grid, resource flag, QA, fs_check and datastore actions - Location.get_data_grid (hdx_datagrid_show) and Location.invalidate_data_grid (invalidate_data_completeness_for_location) - Resource.is_p_coded/set_p_coded (hdx_p_coded_resource_update), is_in_hapi/set_in_hapi (hdx_mark_resource_in_hapi), is_in_quarantine/set_in_quarantine (hdx_mark_resource_in_quarantine), qa_patch (hdx_qa_resource_patch), revise_fs_check (hdx_fs_check_resource_revise), reset_fs_check (hdx_fs_check_resource_reset) and push_to_datastore (hdx_push_resource_to_datastore) - Dataset.get_shape_info (shape_info_show), get_recommended_tags (hdx_recommend_tags), is_allowed_for_datastore (hdx_is_package_allowed_for_datastore), push_to_datastore, set_qa_completed (hdx_mark_qa_completed), qa_revise_resources (hdx_qa_package_revise_resource) and reset_fs_check (hdx_fs_check_package_reset) - hdx_stats.get_user_statistics (hdx_user_statistics) and get_organization_statistics (hdx_organization_statistics) --- documentation/index.md | 79 ++++++++++ src/hdx/api/utilities/hdx_stats.py | 43 +++++- src/hdx/data/dataset.py | 137 +++++++++++++++++ src/hdx/data/location.py | 43 ++++++ src/hdx/data/resource.py | 174 ++++++++++++++++++++++ tests/hdx/api/utilities/test_hdx_stats.py | 34 ++++- tests/hdx/data/test_dataset_noncore.py | 104 +++++++++++++ tests/hdx/data/test_location.py | 65 ++++++++ tests/hdx/data/test_resource.py | 129 ++++++++++++++++ 9 files changed, 804 insertions(+), 4 deletions(-) diff --git a/documentation/index.md b/documentation/index.md index bc6daf6..b08a1d3 100755 --- a/documentation/index.md +++ b/documentation/index.md @@ -647,6 +647,35 @@ This returns a list of dictionaries, each from a resource name to a list of check events with keys such as `state`, `timestamp`, `sheet_changes` and `hxl_proxy_response`. +Similarly, you can get the geopreview shape information of a dataset's +resources: + + shape_info = dataset.get_shape_info() + +HDX can recommend tags for a dataset based on its title and organization +(`owner_org`). Tags are ordered from most to least recommended: + + tags = dataset.get_recommended_tags() + +You can check if the dataset or its organization is in HDX's allowlist for +automatic loading of resources into the datastore: + + allowed = dataset.is_allowed_for_datastore() + +HDX sysadmins can submit all of a dataset's CSV resources to DataPusher+ +(returning False if there are none): + + submitted = dataset.push_to_datastore() + +Users who are sysadmins or have permission to manage QA can mark QA of a +dataset as completed, set a QA field of all its resources or clear a file +structure check field (`fs_check_info` by default) of all its resources. The +dataset is updated from the result: + + dataset.set_qa_completed(True) + dataset.qa_revise_resources("pii", "false") + dataset.reset_fs_check() + ### Time Period Time Period is a mandatory field in HDX. It is the earliest start date and @@ -964,6 +993,32 @@ HDX: events = resource.get_fs_check_info() +Users who are sysadmins or have permission to manage QA can set or clear a +file structure check field of a resource. If the key is `fs_check_info` and the +value is a dictionary, it is appended to the existing check events (the last 20 +are kept): + + resource.revise_fs_check("fs_check_info", {"state": "success", ...}) + resource.reset_fs_check() # key defaults to fs_check_info + +They can also quarantine a resource (which removes its `shape_info` and +geopreview) or take it out of quarantine, and patch other QA fields: + + in_quarantine = resource.is_in_quarantine() + resource.set_in_quarantine(True) + resource.qa_patch(pii="true", sdc_report_flag="false") + +A resource can be flagged as containing p-codes or as being in HAPI. Setting +these flags requires sysadmin or permission to manage p-codes or the in HAPI +flag respectively: + + p_coded = resource.is_p_coded() # None if not set + resource.set_p_coded(True) + in_hapi = resource.is_in_hapi() + resource.set_in_hapi(True) + +These methods update the resource from the result. + ### Data Dictionary A resource can have a data dictionary describing the columns in the data it @@ -1041,6 +1096,11 @@ the latest job or None if there is none: submitted = resource.submit_to_datapusher(ignore_hash=True) status = resource.get_datapusher_status() # eg. status["status"] == "complete" +HDX sysadmins can instead use **push_to_datastore**, which submits the +resource through HDX's wrapper action: + + submitted = resource.push_to_datastore() + To download the whole datastore (rather than the resource's file) use **download_datastore**, which returns the URL and path like **download**. `file_format` can be `csv` (default), `tsv`, `json` or `xml`, and `filename` @@ -1213,6 +1273,17 @@ since Data Grid membership changes over time, rather than a fixed list): This returns a sorted list of 3-letter location codes. +HDX sysadmins can get the Data Grid of a country and clear HDX's cached copy of +it: + + data_grid = Location.get_data_grid("AFG") + Location.invalidate_data_grid("AFG") + +**get_data_grid** returns a dictionary with keys including `title`, `date`, +`stats` and `categories` (or None if not found). Each category has +`subcategories`, each with a list of `datasets`. Complementary datasets have +`is_complementary` set to True. + You can get the datasets belonging to a location as follows: datasets = location.get_datasets(**kwargs) @@ -1298,6 +1369,14 @@ and push them to analytics: pushed_stats = push_general_stats() +HDX sysadmins can get the number of activities of each active user (optionally +between start and end dates) and lists of active and inactive organizations: + + from hdx.api.utilities.hdx_stats import get_organization_statistics, get_user_statistics + + user_stats = get_user_statistics(start_date="2026-01-01", end_date="2026-06-30") + org_stats = get_organization_statistics() + ## Pipeline State The HDXState class allows the reading and writing of state to a given dataset. Input diff --git a/src/hdx/api/utilities/hdx_stats.py b/src/hdx/api/utilities/hdx_stats.py index a104be9..e747a48 100644 --- a/src/hdx/api/utilities/hdx_stats.py +++ b/src/hdx/api/utilities/hdx_stats.py @@ -4,12 +4,15 @@ from hdx.data.hdxobject import HDXError -def _call_stats_action(action: str, configuration: Configuration | None) -> dict: +def _call_stats_action( + action: str, configuration: Configuration | None, data: dict | None = None +) -> dict: """Call an HDX statistics action Args: action: CKAN action name configuration: HDX configuration. Defaults to global configuration. + data: Parameters to pass to the action. Defaults to None. Returns: Statistics dictionary @@ -17,7 +20,7 @@ def _call_stats_action(action: str, configuration: Configuration | None) -> dict if configuration is None: configuration = Configuration.read() try: - return configuration.call_remoteckan(action, {}) + return configuration.call_remoteckan(action, data or {}) except Exception as e: raise HDXError(f"Failed when trying to call {action}!") from e @@ -45,3 +48,39 @@ def push_general_stats(configuration: Configuration | None = None) -> dict: Statistics that were pushed """ return _call_stats_action("hdx_push_general_stats", configuration) + + +def get_user_statistics( + start_date: str | None = None, + end_date: str | None = None, + configuration: Configuration | None = None, +) -> dict: + """Get the number of activities of each active user on HDX, optionally + between start and end dates. Requires sysadmin. + + Args: + start_date: Count activities on or after this date eg. 2026-01-01. Defaults to None. + end_date: Count activities on or before this date. Defaults to None. + configuration: HDX configuration. Defaults to global configuration. + + Returns: + Dictionary from user id to dictionary with keys id, name and no_of_activities + """ + data = {} + if start_date: + data["start_date"] = start_date + if end_date: + data["end_date"] = end_date + return _call_stats_action("hdx_user_statistics", configuration, data) + + +def get_organization_statistics(configuration: Configuration | None = None) -> dict: + """Get active and inactive organizations on HDX. Requires sysadmin. + + Args: + configuration: HDX configuration. Defaults to global configuration. + + Returns: + Dictionary with keys active_organizations and inactive_organizations + """ + return _call_stats_action("hdx_organization_statistics", configuration) diff --git a/src/hdx/data/dataset.py b/src/hdx/data/dataset.py index 9116f7b..64a3607 100755 --- a/src/hdx/data/dataset.py +++ b/src/hdx/data/dataset.py @@ -148,6 +148,13 @@ def actions() -> dict[str, str]: "autocomplete": "package_autocomplete", "create_default_views": "package_create_default_resource_views", "fs_check_info": "fs_check_info_show", + "fs_check_reset": "hdx_fs_check_package_reset", + "shape_info": "shape_info_show", + "recommend_tags": "hdx_recommend_tags", + "allowed_for_datastore": "hdx_is_package_allowed_for_datastore", + "push_to_datastore": "hdx_push_resource_to_datastore", + "qa_completed": "hdx_mark_qa_completed", + "qa_revise_resources": "hdx_qa_package_revise_resource", } def __setitem__(self, key: Any, value: Any) -> None: @@ -2347,6 +2354,136 @@ def get_fs_check_info(self) -> list[dict]: return result return [] + def _set_from_package(self, package: dict) -> None: + """Set the dataset's metadata and resources from a package metadata + dictionary returned by HDX + + Args: + package: Package metadata dictionary + + Returns: + None + """ + self.data = package + self.init_resources() + self.separate_resources() + + def reset_fs_check(self, key: str = "fs_check_info") -> None: + """Clear a file structure check field of all the dataset's resources + in HDX. Requires sysadmin or permission to manage QA. + + Args: + key: Resource field to clear. Defaults to fs_check_info. + + Returns: + None + """ + result = self._write_to_hdx( + "fs_check_reset", {"package_id": self.data["id"], "key": key}, "package_id" + ) + self._set_from_package(result["package"]) + + def set_qa_completed(self, qa_completed: bool) -> None: + """Mark QA of the dataset in HDX as completed or not. Requires + sysadmin or permission to manage QA. + + Args: + qa_completed: Whether QA is completed + + Returns: + None + """ + result = self._write_to_hdx( + "qa_completed", {"id": self.data["id"], "qa_completed": qa_completed}, "id" + ) + self._set_from_package(result["package"]) + + def qa_revise_resources(self, key: str, value: Any) -> None: + """Set a QA field of all the dataset's resources in HDX. Requires + sysadmin or permission to manage QA. + + Args: + key: Resource field eg. pii + value: Value to set + + Returns: + None + """ + result = self._write_to_hdx( + "qa_revise_resources", + {"id": self.data["id"], "key": key, "value": value}, + "id", + ) + self._set_from_package(result["package"]) + + def get_shape_info(self) -> list[dict]: + """Get geopreview shape information for the dataset's resources from + HDX + + Returns: + List of dictionaries from resource name to shape information + """ + success, result = self._read_from_hdx( + "dataset", + self.get_name_or_id(prefer_name=False), + "id", + self.actions()["shape_info"], + ) + if success: + return result + return [] + + def get_recommended_tags(self) -> list[str]: + """Get tags recommended by HDX for the dataset based on its title and + organization. Tags are ordered from most to least recommended. + + Returns: + List of recommended tags + """ + data = {"title": self.data.get("title")} + organization = self.data.get("owner_org") + if organization: + data["organization"] = organization + try: + result = self.configuration.call_remoteckan( + self.actions()["recommend_tags"], data + ) + except Exception as e: + raise HDXError("Failed when trying to get recommended tags!") from e + return [tag["name"] for tag in result] + + def is_allowed_for_datastore(self) -> bool: + """Return if the dataset or its organization is in HDX's allowlist for + automatic loading of resources into the datastore + + Returns: + Whether dataset is allowed for datastore + """ + success, result = self._read_from_hdx( + "dataset", + self.data["id"], + "package_id", + self.actions()["allowed_for_datastore"], + ) + if success: + return result + logger.debug(result) + return False + + def push_to_datastore(self) -> bool: + """Submit all the dataset's CSV resources to DataPusher+ using HDX's + wrapper action. Requires sysadmin. + + Returns: + True if submitted, False if not (eg. no CSV resources) + """ + result = self._write_to_hdx( + "push_to_datastore", {"dataset_id": self.data["id"]}, "dataset_id" + ) + if not result["success"]: + logger.debug(result["message"]) + return result["success"] + def generate_resource( self, folder: Path | str, diff --git a/src/hdx/data/location.py b/src/hdx/data/location.py index 7b4e14d..27457e1 100644 --- a/src/hdx/data/location.py +++ b/src/hdx/data/location.py @@ -43,6 +43,8 @@ def actions() -> dict[str, str]: "delete": "group_delete", "list": "group_list", "autocomplete": "group_autocomplete", + "data_grid": "hdx_datagrid_show", + "invalidate_data_grid": "invalidate_data_completeness_for_location", } @classmethod @@ -218,6 +220,47 @@ def get_data_grid_countries( if len(group["name"]) == 3 and group.get("data_completeness") == "active" ) + @staticmethod + def get_data_grid( + iso3: str, configuration: Configuration | None = None + ) -> dict | None: + """Get the Data Grid of a country. The dictionary has keys including + title, date, stats and categories. Each category has subcategories, + each with a list of datasets. Complementary datasets have + is_complementary set to True. Requires sysadmin. + + Args: + iso3: Country ISO 3-letter code + configuration: HDX configuration. Defaults to global configuration. + + Returns: + Data Grid dictionary or None if not found + """ + location = Location(configuration=configuration) + success, result = location._read_from_hdx( + "location", iso3.lower(), "id", location.actions()["data_grid"] + ) + if success: + return result + logger.debug(result) + return None + + @staticmethod + def invalidate_data_grid( + iso3: str, configuration: Configuration | None = None + ) -> None: + """Clear HDX's cached Data Grid of a country. Requires sysadmin. + + Args: + iso3: Country ISO 3-letter code + configuration: HDX configuration. Defaults to global configuration. + + Returns: + None + """ + location = Location(configuration=configuration) + location._write_to_hdx("invalidate_data_grid", {"name": iso3.lower()}, "name") + def get_activities(self, all_pages: bool = False, **kwargs: Any) -> list[Activity]: """Get the activity stream of this location. One page of results (whose size is given by limit) is returned unless all_pages is True. diff --git a/src/hdx/data/resource.py b/src/hdx/data/resource.py index 39a3e6c..00ca983 100755 --- a/src/hdx/data/resource.py +++ b/src/hdx/data/resource.py @@ -74,6 +74,13 @@ def actions() -> dict[str, str]: "datastore_search_sql": "datastore_search_sql", "datapusher_submit": "datapusher_submit", "datapusher_status": "datapusher_status", + "push_to_datastore": "hdx_push_resource_to_datastore", + "p_coded": "hdx_p_coded_resource_update", + "in_hapi": "hdx_mark_resource_in_hapi", + "in_quarantine": "hdx_mark_resource_in_quarantine", + "qa_patch": "hdx_qa_resource_patch", + "fs_check_revise": "hdx_fs_check_resource_revise", + "fs_check_reset": "hdx_fs_check_resource_reset", } def update_from_yaml( @@ -1113,6 +1120,20 @@ def submit_to_datapusher(self, ignore_hash: bool = False) -> bool: "resource_id", ) + def push_to_datastore(self) -> bool: + """Submit the resource to DataPusher+ using HDX's wrapper action. + Unlike submit_to_datapusher, this requires sysadmin. + + Returns: + True if submitted, False if not + """ + result = self._write_to_hdx( + "push_to_datastore", {"resource_id": self.data["id"]}, "resource_id" + ) + if not result["success"]: + logger.debug(result["message"]) + return result["success"] + def get_datapusher_status(self) -> dict | None: """Get the status of the resource's latest DataPusher+ job @@ -1304,6 +1325,159 @@ def get_fs_check_info(self) -> list[dict]: return json.loads(fs_check_info) return fs_check_info + def _set_from_package(self, package: dict) -> None: + """Set the resource's metadata from the matching resource in a package + metadata dictionary returned by HDX + + Args: + package: Package metadata dictionary + + Returns: + None + """ + for resource in package.get("resources", []): + if resource["id"] == self.data["id"]: + self.data = resource + return + + def revise_fs_check(self, key: str, value: Any) -> None: + """Set a file structure check field of the resource in HDX. If key is + fs_check_info and value is a dictionary, it is appended to the + existing check events (the last 20 are kept) rather than replacing + them. Requires sysadmin or permission to manage QA. + + Args: + key: Resource field eg. fs_check_info + value: Value to set + + Returns: + None + """ + result = self._write_to_hdx( + "fs_check_revise", + { + "package_id": self.data["package_id"], + "id": self.data["id"], + "key": key, + "value": value, + }, + "id", + ) + self._set_from_package(result["package"]) + + def reset_fs_check(self, key: str = "fs_check_info") -> None: + """Clear a file structure check field of the resource in HDX. Requires + sysadmin or permission to manage QA. + + Args: + key: Resource field to clear. Defaults to fs_check_info. + + Returns: + None + """ + result = self._write_to_hdx( + "fs_check_reset", + {"package_id": self.data["package_id"], "id": self.data["id"], "key": key}, + "id", + ) + self._set_from_package(result["package"]) + + def is_p_coded(self) -> bool | None: + """Return if the resource is marked as containing p-codes + + Returns: + Whether resource is p-coded or None if not set + """ + p_coded = self.data.get("p_coded") + if p_coded is None: + return None + if isinstance(p_coded, str): + return p_coded.lower() == "true" + return p_coded + + def set_p_coded(self, p_coded: bool) -> None: + """Mark the resource in HDX as containing p-codes or not. Requires + sysadmin or permission to manage p-codes. + + Args: + p_coded: Whether resource is p-coded + + Returns: + None + """ + self.data = self._write_to_hdx( + "p_coded", {"id": self.data["id"], "p_coded": p_coded}, "id" + ) + + def is_in_hapi(self) -> bool: + """Return if the resource is marked as being in HAPI + + Returns: + Whether resource is in HAPI + """ + return self.data.get("in_hapi") == "yes" + + def set_in_hapi(self, in_hapi: bool) -> None: + """Mark the resource in HDX as being in HAPI or not. Requires sysadmin + or permission to manage the in HAPI flag. + + Args: + in_hapi: Whether resource is in HAPI + + Returns: + None + """ + self.data = self._write_to_hdx( + "in_hapi", + {"id": self.data["id"], "in_hapi": "yes" if in_hapi else "no-data"}, + "id", + ) + + def is_in_quarantine(self) -> bool: + """Return if the resource is in quarantine + + Returns: + Whether resource is in quarantine + """ + in_quarantine = self.data.get("in_quarantine", False) + if isinstance(in_quarantine, str): + return in_quarantine.lower() == "true" + return in_quarantine + + def set_in_quarantine(self, in_quarantine: bool) -> None: + """Put the resource in HDX in quarantine or take it out. Quarantining + also removes the resource's shape_info and geopreview. Requires + sysadmin or permission to manage QA. + + Args: + in_quarantine: Whether resource is in quarantine + + Returns: + None + """ + self.data = self._write_to_hdx( + "in_quarantine", + { + "id": self.data["id"], + "in_quarantine": "true" if in_quarantine else "false", + }, + "id", + ) + + def qa_patch(self, **kwargs: Any) -> None: + """Patch QA fields of the resource in HDX eg. pii, sdc_report_flag, + broken_link. Requires sysadmin or permission to manage QA. + + Args: + **kwargs: Resource fields to patch + + Returns: + None + """ + self.data = self._write_to_hdx( + "qa_patch", {"id": self.data["id"], **kwargs}, "id" + ) + def is_marked_data_updated(self) -> bool: """Return if the resource's data is marked to be updated diff --git a/tests/hdx/api/utilities/test_hdx_stats.py b/tests/hdx/api/utilities/test_hdx_stats.py index 110a875..612ad98 100644 --- a/tests/hdx/api/utilities/test_hdx_stats.py +++ b/tests/hdx/api/utilities/test_hdx_stats.py @@ -6,7 +6,12 @@ from ... import MockResponse from hdx.api.configuration import Configuration -from hdx.api.utilities.hdx_stats import get_general_statistics, push_general_stats +from hdx.api.utilities.hdx_stats import ( + get_general_statistics, + get_organization_statistics, + get_user_statistics, + push_general_stats, +) from hdx.data.hdxobject import HDXError general_statistics = { @@ -17,17 +22,30 @@ pushed_stats = {"mixpanel_meta": {"datasets total": 100, "orgs total": 30}} +user_statistics = {"USER1": {"id": "USER1", "name": "user1", "no_of_activities": 5}} + +organization_statistics = { + "active_organizations": [{"id": "ORG1", "name": "org1"}], + "inactive_organizations": [], +} + class TestHDXStats: succeed = True + data = None @pytest.fixture(scope="function") def post_stats(self): class MockSession: @staticmethod def post(url, data, **kwargs): + TestHDXStats.data = json.loads(data.decode("utf-8")) if TestHDXStats.succeed: - if "hdx_general_statistics" in url: + if "hdx_user_statistics" in url: + result = json.dumps(user_statistics) + elif "hdx_organization_statistics" in url: + result = json.dumps(organization_statistics) + elif "hdx_general_statistics" in url: result = json.dumps(general_statistics) elif "hdx_push_general_stats" in url: result = json.dumps(pushed_stats) @@ -49,6 +67,18 @@ def test_get_general_statistics(self, configuration, post_stats): assert get_general_statistics() == general_statistics assert get_general_statistics(Configuration.read()) == general_statistics + def test_get_user_statistics(self, configuration, post_stats): + assert get_user_statistics() == user_statistics + assert TestHDXStats.data == {} + assert get_user_statistics("2026-01-01", "2026-06-30") == user_statistics + assert TestHDXStats.data == { + "start_date": "2026-01-01", + "end_date": "2026-06-30", + } + + def test_get_organization_statistics(self, configuration, post_stats): + assert get_organization_statistics() == organization_statistics + def test_push_general_stats(self, configuration, post_stats): assert push_general_stats() == pushed_stats TestHDXStats.succeed = False diff --git a/tests/hdx/data/test_dataset_noncore.py b/tests/hdx/data/test_dataset_noncore.py index abcb8ae..fcf26d1 100755 --- a/tests/hdx/data/test_dataset_noncore.py +++ b/tests/hdx/data/test_dataset_noncore.py @@ -177,6 +177,110 @@ def test_get_fs_check_info(self, configuration, post_fs_check_info): dataset = Dataset({"id": "TEST2"}) assert dataset.get_fs_check_info() == [] + @pytest.fixture(scope="function") + def post_dataset_actions(self): + calls = [] + + class MockSession: + @staticmethod + def post(url, data, **kwargs): + datadict = json.loads(data.decode("utf-8")) + action = url.split("/")[-1] + calls.append((action, datadict)) + package = { + "id": "TEST1", + "name": "test-dataset", + "resources": [{"id": "RES1", "name": "MyResource1"}], + } + if datadict.get("id", datadict.get("package_id")) == "NOTFOUND": + return MockResponse( + 404, + f'{{"success": false, "error": {{"message": "Not found", "__type": "Not Found Error"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + if action == "shape_info_show": + result = json.dumps([{"MyResource1": {"state": "success"}}]) + elif action == "hdx_recommend_tags": + result = json.dumps( + [ + {"name": "health", "count": 5}, + {"name": "cholera", "count": 2}, + ] + ) + elif action == "hdx_is_package_allowed_for_datastore": + result = "true" + elif action == "hdx_push_resource_to_datastore": + result = '{"success": false, "message": "No CSV resources found for dataset"}' + elif action == "hdx_mark_qa_completed": + package["qa_completed"] = datadict["qa_completed"] + result = json.dumps({"package": package}) + elif action in ( + "hdx_qa_package_revise_resource", + "hdx_fs_check_package_reset", + ): + package["resources"][0][datadict["key"]] = datadict.get("value", "") + result = json.dumps({"package": package}) + else: + return MockResponse( + 404, + f'{{"success": false, "error": {{"message": "TEST ERROR: Unexpected {action}", "__type": "TEST ERROR"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + return MockResponse( + 200, + f'{{"success": true, "result": {result}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + + Configuration.read().remoteckan().session = MockSession() + return calls + + def test_read_helpers(self, configuration, post_dataset_actions): + dataset = Dataset( + {"id": "TEST1", "title": "Cholera cases", "owner_org": "ORG1"} + ) + assert dataset.get_shape_info() == [{"MyResource1": {"state": "success"}}] + assert dataset.get_recommended_tags() == ["health", "cholera"] + assert post_dataset_actions[-1] == ( + "hdx_recommend_tags", + {"title": "Cholera cases", "organization": "ORG1"}, + ) + assert dataset.is_allowed_for_datastore() is True + assert post_dataset_actions[-1] == ( + "hdx_is_package_allowed_for_datastore", + {"package_id": "TEST1"}, + ) + assert dataset.push_to_datastore() is False + assert post_dataset_actions[-1] == ( + "hdx_push_resource_to_datastore", + {"dataset_id": "TEST1"}, + ) + dataset = Dataset({"id": "NOTFOUND"}) + assert dataset.get_shape_info() == [] + assert dataset.is_allowed_for_datastore() is False + + def test_qa(self, configuration, post_dataset_actions): + dataset = Dataset({"id": "TEST1"}) + dataset.set_qa_completed(True) + assert post_dataset_actions[-1] == ( + "hdx_mark_qa_completed", + {"id": "TEST1", "qa_completed": True}, + ) + assert dataset["qa_completed"] is True + assert dataset.get_resource()["id"] == "RES1" + dataset.qa_revise_resources("pii", "false") + assert post_dataset_actions[-1] == ( + "hdx_qa_package_revise_resource", + {"id": "TEST1", "key": "pii", "value": "false"}, + ) + assert dataset.get_resource()["pii"] == "false" + dataset.reset_fs_check() + assert post_dataset_actions[-1] == ( + "hdx_fs_check_package_reset", + {"package_id": "TEST1", "key": "fs_check_info"}, + ) + assert dataset.get_resource()["fs_check_info"] == "" + dataset = Dataset({"id": "NOTFOUND"}) + with pytest.raises(HDXError): + dataset.set_qa_completed(True) + def test_get_name_or_id(self, configuration, hdx_config_yaml, project_config_yaml): dataset = Dataset() assert dataset.get_name_or_id() is None diff --git a/tests/hdx/data/test_location.py b/tests/hdx/data/test_location.py index 9231f86..f8d3bb0 100644 --- a/tests/hdx/data/test_location.py +++ b/tests/hdx/data/test_location.py @@ -100,6 +100,29 @@ def mockgetdatasets(url, datadict): ) +data_grid = { + "iso3": "afg", + "title": "Afghanistan", + "date": "2026-10-01", + "stats": {"complete": 10, "incomplete": 2, "empty": 1}, + "categories": [ + { + "title": "Health", + "stats": {"complete": 1}, + "subcategories": [ + { + "title": "Health facilities", + "stats": {"complete": 1}, + "datasets": [ + {"id": "TEST1", "name": "afg-health", "is_complementary": False} + ], + } + ], + } + ], +} + + class TestLocation: @pytest.fixture(scope="function") def read(self): @@ -280,6 +303,36 @@ def post(url, data, **kwargs): Configuration.read().remoteckan().session = MockSession() + @pytest.fixture(scope="function") + def post_data_grid(self): + calls = [] + + class MockSession: + @staticmethod + def post(url, data, **kwargs): + datadict = json.loads(data.decode("utf-8")) + action = url.split("/")[-1] + calls.append((action, datadict)) + if action == "hdx_datagrid_show" and datadict["id"] == "afg": + result = json.dumps(data_grid) + elif ( + action == "invalidate_data_completeness_for_location" + and datadict["name"] == "afg" + ): + result = '{"message": "Successfully invalidated data completeness cache for afg"}' + else: + return MockResponse( + 404, + f'{{"success": false, "error": {{"message": "Not found", "__type": "Not Found Error"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + return MockResponse( + 200, + f'{{"success": true, "result": {result}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + + Configuration.read().remoteckan().session = MockSession() + return calls + def test_read_from_hdx(self, configuration, read, mocksmtp): location = Location.read_from_hdx("f8b0e58f-9be5-4b25-9d5f-1c8b6d8b6a0a") assert location["id"] == "f8b0e58f-9be5-4b25-9d5f-1c8b6d8b6a0a" @@ -379,3 +432,15 @@ def test_autocomplete(self, configuration, post_autocomplete): def test_get_data_grid_countries(self, configuration, post_all_fields): countries = Location.get_data_grid_countries() assert countries == ["afg", "gnq"] + + def test_data_grid(self, configuration, post_data_grid): + assert Location.get_data_grid("AFG") == data_grid + assert post_data_grid[-1] == ("hdx_datagrid_show", {"id": "afg"}) + assert Location.get_data_grid("xyz") is None + Location.invalidate_data_grid("AFG") + assert post_data_grid[-1] == ( + "invalidate_data_completeness_for_location", + {"name": "afg"}, + ) + with pytest.raises(HDXError): + Location.invalidate_data_grid("xyz") diff --git a/tests/hdx/data/test_resource.py b/tests/hdx/data/test_resource.py index 6229422..672b61f 100755 --- a/tests/hdx/data/test_resource.py +++ b/tests/hdx/data/test_resource.py @@ -811,6 +811,59 @@ def post(url, data, **kwargs): Configuration.read().remoteckan().session = MockSession() return calls + @pytest.fixture(scope="function") + def post_qa(self): + calls = [] + + class MockSession: + @staticmethod + def post(url, data, **kwargs): + datadict = json.loads(data.decode("utf-8")) + action = url.split("/")[-1] + calls.append((action, datadict)) + if datadict.get("id") == "NOTFOUND": + return MockResponse( + 404, + f'{{"success": false, "error": {{"message": "Not found", "__type": "Not Found Error"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + if action in ( + "hdx_fs_check_resource_revise", + "hdx_fs_check_resource_reset", + ): + package = { + "id": datadict["package_id"], + "resources": [ + {"id": "OTHER"}, + { + "id": datadict["id"], + "package_id": datadict["package_id"], + datadict["key"]: datadict.get("value", ""), + }, + ], + } + result = json.dumps({"package": package}) + elif action in ( + "hdx_p_coded_resource_update", + "hdx_mark_resource_in_hapi", + "hdx_mark_resource_in_quarantine", + "hdx_qa_resource_patch", + ): + result = json.dumps(datadict) + elif action == "hdx_push_resource_to_datastore": + result = '{"success": true, "message": "Resource submitted to Datapusher Plus"}' + else: + return MockResponse( + 404, + f'{{"success": false, "error": {{"message": "TEST ERROR: Unexpected {action}", "__type": "TEST ERROR"}}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + return MockResponse( + 200, + f'{{"success": true, "result": {result}, "help": "http://test-data.humdata.org/api/3/action/help_show?name={action}"}}', + ) + + Configuration.read().remoteckan().session = MockSession() + return calls + def test_read_from_hdx(self, configuration, read): resource = Resource.read_from_hdx("74b74ae1-df0c-4716-829f-4f939a046811") assert resource["id"] == "de6549d8-268b-4dfe-adaf-a4ae5c8510d5" @@ -1426,6 +1479,82 @@ def test_datapusher(self, configuration, post_datapusher): with pytest.raises(HDXError): resource.submit_to_datapusher() + def test_flags_and_qa(self, configuration, post_qa): + resource = Resource({"id": "RES1", "package_id": "PKG1"}) + assert resource.is_p_coded() is None + resource.set_p_coded(True) + assert post_qa[-1] == ( + "hdx_p_coded_resource_update", + {"id": "RES1", "p_coded": True}, + ) + assert resource.is_p_coded() is True + resource["p_coded"] = "False" + assert resource.is_p_coded() is False + + assert resource.is_in_hapi() is False + resource.set_in_hapi(True) + assert post_qa[-1] == ( + "hdx_mark_resource_in_hapi", + {"id": "RES1", "in_hapi": "yes"}, + ) + assert resource.is_in_hapi() is True + resource.set_in_hapi(False) + assert post_qa[-1][1]["in_hapi"] == "no-data" + assert resource.is_in_hapi() is False + + assert resource.is_in_quarantine() is False + resource.set_in_quarantine(True) + assert post_qa[-1] == ( + "hdx_mark_resource_in_quarantine", + {"id": "RES1", "in_quarantine": "true"}, + ) + assert resource.is_in_quarantine() is True + resource.set_in_quarantine(False) + assert resource.is_in_quarantine() is False + + resource.qa_patch(pii="true", sdc_report_flag="false") + assert post_qa[-1] == ( + "hdx_qa_resource_patch", + {"id": "RES1", "pii": "true", "sdc_report_flag": "false"}, + ) + assert resource["pii"] == "true" + + resource = Resource({"id": "RES1", "package_id": "PKG1"}) + event = {"state": "success", "timestamp": "2026-10-01T10:00:00"} + resource.revise_fs_check("fs_check_info", event) + assert post_qa[-1] == ( + "hdx_fs_check_resource_revise", + { + "package_id": "PKG1", + "id": "RES1", + "key": "fs_check_info", + "value": event, + }, + ) + assert resource.data == { + "id": "RES1", + "package_id": "PKG1", + "fs_check_info": event, + } + resource.reset_fs_check() + assert post_qa[-1] == ( + "hdx_fs_check_resource_reset", + {"package_id": "PKG1", "id": "RES1", "key": "fs_check_info"}, + ) + assert resource["fs_check_info"] == "" + + assert resource.push_to_datastore() is True + assert post_qa[-1] == ( + "hdx_push_resource_to_datastore", + {"resource_id": "RES1"}, + ) + + resource = Resource({"id": "NOTFOUND", "package_id": "PKG1"}) + with pytest.raises(HDXError): + resource.set_p_coded(True) + with pytest.raises(HDXError): + resource.set_in_quarantine(True) + def test_update_hdx_data_dictionary(self, configuration, post_datapusher): existing = [ {