diff --git a/docs/hamilton-ui/ui.rst b/docs/hamilton-ui/ui.rst index 3e1811eba..708aa44ca 100644 --- a/docs/hamilton-ui/ui.rst +++ b/docs/hamilton-ui/ui.rst @@ -285,6 +285,9 @@ Changing behavior of what is captured ------------------------------------- By default, a lot is captured and sent to the Apache Hamilton UI. +Calendar dates, times, and durations appear under Untyped Columns. The tracker +reports their data type and counts, while Datetime Columns show timestamp statistics. + Here are a few options that can change that - these can be found in `hamilton_sdk.tracking.constants`. You can either change the defaults by directly changing the constants, by specifying them in a config file, or via environment variables. diff --git a/ui/sdk/src/hamilton_sdk/tracking/pandas_stats.py b/ui/sdk/src/hamilton_sdk/tracking/pandas_stats.py index 10994dde3..1e258495b 100644 --- a/ui/sdk/src/hamilton_sdk/tracking/pandas_stats.py +++ b/ui/sdk/src/hamilton_sdk/tracking/pandas_stats.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +from datetime import date from typing import Any import pandas as pd @@ -53,13 +54,10 @@ def _compute_stats(df: pd.DataFrame) -> dict[str, dict[str, Any]]: """ category_types = df.select_dtypes(include=["category"]) string_types = df.select_dtypes(include=["string"]) - numeric_types = df.select_dtypes(include=["number"]) + # Timedeltas are durations and have no renderer in schema v0.0.3. + numeric_types = df.select_dtypes(include=["number"], exclude=["timedelta"]) bool_types = df.select_dtypes(include=["bool"]) - object_types = df.select_dtypes(include=["object"]) - datetime_types = df.select_dtypes(include=["datetime", "timedelta"]) - unhandled_types = df.select_dtypes( - exclude=["string", "category", "number", "bool", "object", "datetime", "timedelta"] - ) + datetime_types = df.select_dtypes(include=["datetime"]) column_order = {col: index for index, col in enumerate(df.columns)} stats = {} @@ -92,12 +90,15 @@ def execute_col( stats[col] = execute_col("numeric_column_stats", numeric_types[col], col, column_order[col]) for col in bool_types.columns: stats[col] = execute_col("boolean_column_stats", bool_types[col], col, column_order[col]) - for col in datetime_types.columns: - stats[col] = execute_col( - "datetime_column_stats", datetime_types[col], col, column_order[col] - ) - for col in list(object_types.columns) + list(unhandled_types.columns): - stats[col] = execute_col("unhandled_column_stats", df[col], col, column_order[col]) + for col, dtype in datetime_types.dtypes.items(): + # Pandas can include Arrow calendar dates in its datetime selection. + if dtype.type is not date: + stats[col] = execute_col( + "datetime_column_stats", datetime_types[col], col, column_order[col] + ) + for col, position in column_order.items(): + if col not in stats: + stats[col] = execute_col("unhandled_column_stats", df[col], col, position) return stats diff --git a/ui/sdk/src/hamilton_sdk/tracking/polars_stats.py b/ui/sdk/src/hamilton_sdk/tracking/polars_stats.py index 76ca860b8..7a6523816 100644 --- a/ui/sdk/src/hamilton_sdk/tracking/polars_stats.py +++ b/ui/sdk/src/hamilton_sdk/tracking/polars_stats.py @@ -57,8 +57,9 @@ def _compute_stats(df: pl.DataFrame) -> dict[str, dict[str, Any]]: numeric_types = df.select(selectors.numeric()) bool_types = df.select([pl.col(pl.Boolean)]) # df.select([pl.col(pl.Object)]) - # Time/Duration are not valid JS Dates in the UI; leave them for unhandled stats. - date_types = df.select(selectors.date() | selectors.datetime()) + # The datetime renderer interprets dates as UTC midnight, shifting calendar days + # in some time zones. Dates, times and durations use unhandled stats in v0.0.3. + datetime_types = df.select(selectors.datetime()) # get all other columns that have not been selected # df.select( # ~cs.by_dtype([pl.Categorical, pl.Utf8, pl.Boolean, pl.Object]) @@ -93,8 +94,10 @@ def execute_col(target_output: str, col: pl.Series, name: str, position: int) -> stats[col] = execute_col("numeric_column_stats", numeric_types[col], col, column_order[col]) for col in bool_types.columns: stats[col] = execute_col("boolean_column_stats", bool_types[col], col, column_order[col]) - for col in date_types.columns: - stats[col] = execute_col("datetime_column_stats", date_types[col], col, column_order[col]) + for col in datetime_types.columns: + stats[col] = execute_col( + "datetime_column_stats", datetime_types[col], col, column_order[col] + ) for col, _position in column_order.items(): if col not in stats: stats[col] = execute_col("unhandled_column_stats", df[col], col, column_order[col]) diff --git a/ui/sdk/tests/tracking/test_pandas_stats.py b/ui/sdk/tests/tracking/test_pandas_stats.py index 9a60401b7..25f6c36f7 100644 --- a/ui/sdk/tests/tracking/test_pandas_stats.py +++ b/ui/sdk/tests/tracking/test_pandas_stats.py @@ -15,7 +15,11 @@ # specific language governing permissions and limitations # under the License. +import json +from datetime import date + import pandas as pd +import pytest from hamilton_sdk.tracking import pandas_stats as ps @@ -227,3 +231,34 @@ def test_compute_stats_df(): }, } assert actual == expected_stats + + +def test_compute_stats_timedelta_series_is_unhandled(): + series = pd.Series( + [pd.Timedelta(days=-1), None, pd.Timedelta(days=2)], + name="duration", + dtype="timedelta64[ns]", + ) + + actual = ps.compute_stats_series(series, "test", {}) + + assert actual["observability_value"]["duration"] == { + "base_data_type": "unhandled", + "count": 3, + "data_type": "timedelta64[ns]", + "missing": 1, + "name": "duration", + "pos": 0, + } + json.dumps(actual, allow_nan=False) + + +def test_compute_stats_arrow_date_series_is_unhandled(): + pa = pytest.importorskip("pyarrow") + series = pd.Series([date(2024, 1, 1), None], name="day", dtype=pd.ArrowDtype(pa.date32())) + + actual = ps.compute_stats_series(series, "test", {}) + + assert actual["observability_value"]["day"]["base_data_type"] == "unhandled" + assert actual["observability_value"]["day"]["missing"] == 1 + json.dumps(actual, allow_nan=False) diff --git a/ui/sdk/tests/tracking/test_polars_stats.py b/ui/sdk/tests/tracking/test_polars_stats.py index be681c21a..eaff6c49f 100644 --- a/ui/sdk/tests/tracking/test_polars_stats.py +++ b/ui/sdk/tests/tracking/test_polars_stats.py @@ -150,25 +150,12 @@ def test_compute_stats_df(): "pos": 6, }, "h": { - "base_data_type": "datetime", + "base_data_type": "unhandled", "count": 5, "data_type": "Date", - "histogram": {}, - "max": "2024-01-14", - "mean": "2024-01-06T19:12:00", - "min": "2024-01-01", "missing": 0, "name": "h", "pos": 7, - "quantiles": { - 0.1: "2024-01-01T00:00:00", - 0.25: "2024-01-03T00:00:00", - 0.5: "2024-01-06T00:00:00", - 0.75: "2024-01-10T00:00:00", - 0.9: "2024-01-14T00:00:00", - }, - "std": 0.0, - "zeros": 0, }, "i": { "base_data_type": "numeric", @@ -186,33 +173,15 @@ def test_compute_stats_df(): "zeros": 0, }, "j": { - "base_data_type": "datetime", + "base_data_type": "unhandled", "count": 5, "data_type": "Date", - "histogram": {}, - "max": "2022-05-01", - "mean": "2022-03-02T00:00:00", - "min": "2022-01-01", "missing": 0, "name": "j", "pos": 9, - "quantiles": { - 0.1: "2022-01-01T00:00:00", - 0.25: "2022-02-01T00:00:00", - 0.5: "2022-03-01T00:00:00", - 0.75: "2022-04-01T00:00:00", - 0.9: "2022-05-01T00:00:00", - }, - "std": 0.0, - "zeros": 0, }, }, } - # Date quantile behavior varies across polars versions — some return populated values, - # others return empty {}. Don't assert on quantiles for Date columns. - for col in ("h", "j"): - actual["observability_value"][col].pop("quantiles", None) - expected_stats["observability_value"][col].pop("quantiles", None) assert actual == expected_stats diff --git a/ui/sdk/tests/tracking/test_runs.py b/ui/sdk/tests/tracking/test_runs.py index 2859b85d9..39a16f133 100644 --- a/ui/sdk/tests/tracking/test_runs.py +++ b/ui/sdk/tests/tracking/test_runs.py @@ -510,8 +510,17 @@ def create_node(name: str, type_: type) -> node.Node: ( pl.date_range(date(2023, 1, 1), date(2023, 1, 3), eager=True).alias("a"), create_node("a", pl.Series), - "dagworks_describe", # TODO: fix quantiles - {"a": {"base_data_type": "unknown-polars", "data_type": "Date", "name": "a", "pos": 0}}, + "dagworks_describe", + { + "a": { + "base_data_type": "unhandled", + "count": 3, + "data_type": "Date", + "missing": 0, + "name": "a", + "pos": 0, + } + }, ), ], ids=[ @@ -531,7 +540,7 @@ def create_node(name: str, type_: type) -> node.Node: "polars_df", "polars_series", "pl_nan_series", - "pl_datetime_series", + "pl_date_series", ], ) def test_process_result_happy(test_result, test_node, observability_type, stats):