Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions docs/hamilton-ui/ui.rst
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,9 @@ Changing behavior of what is captured
-------------------------------------
By default, a lot is captured and sent to the Apache Hamilton UI.

Calendar dates, times, and durations appear under Untyped Columns. The tracker
reports their data type and counts, while Datetime Columns show timestamp statistics.

Here are a few options that can change that - these can be found
in `hamilton_sdk.tracking.constants`. You can either change the defaults by
directly changing the constants, by specifying them in a config file, or via environment variables.
Expand Down
25 changes: 13 additions & 12 deletions ui/sdk/src/hamilton_sdk/tracking/pandas_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.

from datetime import date
from typing import Any

import pandas as pd
Expand Down Expand Up @@ -53,13 +54,10 @@ def _compute_stats(df: pd.DataFrame) -> dict[str, dict[str, Any]]:
"""
category_types = df.select_dtypes(include=["category"])
string_types = df.select_dtypes(include=["string"])
numeric_types = df.select_dtypes(include=["number"])
# Timedeltas are durations and have no renderer in schema v0.0.3.
numeric_types = df.select_dtypes(include=["number"], exclude=["timedelta"])
bool_types = df.select_dtypes(include=["bool"])
object_types = df.select_dtypes(include=["object"])
datetime_types = df.select_dtypes(include=["datetime", "timedelta"])
unhandled_types = df.select_dtypes(
exclude=["string", "category", "number", "bool", "object", "datetime", "timedelta"]
)
datetime_types = df.select_dtypes(include=["datetime"])
column_order = {col: index for index, col in enumerate(df.columns)}
stats = {}

Expand Down Expand Up @@ -92,12 +90,15 @@ def execute_col(
stats[col] = execute_col("numeric_column_stats", numeric_types[col], col, column_order[col])
for col in bool_types.columns:
stats[col] = execute_col("boolean_column_stats", bool_types[col], col, column_order[col])
for col in datetime_types.columns:
stats[col] = execute_col(
"datetime_column_stats", datetime_types[col], col, column_order[col]
)
for col in list(object_types.columns) + list(unhandled_types.columns):
stats[col] = execute_col("unhandled_column_stats", df[col], col, column_order[col])
for col, dtype in datetime_types.dtypes.items():
# Pandas can include Arrow calendar dates in its datetime selection.
if dtype.type is not date:
stats[col] = execute_col(
"datetime_column_stats", datetime_types[col], col, column_order[col]
)
for col, position in column_order.items():
if col not in stats:
stats[col] = execute_col("unhandled_column_stats", df[col], col, position)
return stats


Expand Down
11 changes: 7 additions & 4 deletions ui/sdk/src/hamilton_sdk/tracking/polars_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -57,8 +57,9 @@ def _compute_stats(df: pl.DataFrame) -> dict[str, dict[str, Any]]:
numeric_types = df.select(selectors.numeric())
bool_types = df.select([pl.col(pl.Boolean)])
# df.select([pl.col(pl.Object)])
# Time/Duration are not valid JS Dates in the UI; leave them for unhandled stats.
date_types = df.select(selectors.date() | selectors.datetime())
# The datetime renderer interprets dates as UTC midnight, shifting calendar days
# in some time zones. Dates, times and durations use unhandled stats in v0.0.3.
datetime_types = df.select(selectors.datetime())
# get all other columns that have not been selected
# df.select(
# ~cs.by_dtype([pl.Categorical, pl.Utf8, pl.Boolean, pl.Object])
Expand Down Expand Up @@ -93,8 +94,10 @@ def execute_col(target_output: str, col: pl.Series, name: str, position: int) ->
stats[col] = execute_col("numeric_column_stats", numeric_types[col], col, column_order[col])
for col in bool_types.columns:
stats[col] = execute_col("boolean_column_stats", bool_types[col], col, column_order[col])
for col in date_types.columns:
stats[col] = execute_col("datetime_column_stats", date_types[col], col, column_order[col])
for col in datetime_types.columns:
stats[col] = execute_col(
"datetime_column_stats", datetime_types[col], col, column_order[col]
)
for col, _position in column_order.items():
if col not in stats:
stats[col] = execute_col("unhandled_column_stats", df[col], col, column_order[col])
Expand Down
35 changes: 35 additions & 0 deletions ui/sdk/tests/tracking/test_pandas_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,11 @@
# specific language governing permissions and limitations
# under the License.

import json
from datetime import date

import pandas as pd
import pytest

from hamilton_sdk.tracking import pandas_stats as ps

Expand Down Expand Up @@ -227,3 +231,34 @@ def test_compute_stats_df():
},
}
assert actual == expected_stats


def test_compute_stats_timedelta_series_is_unhandled():
series = pd.Series(
[pd.Timedelta(days=-1), None, pd.Timedelta(days=2)],
name="duration",
dtype="timedelta64[ns]",
)

actual = ps.compute_stats_series(series, "test", {})

assert actual["observability_value"]["duration"] == {
"base_data_type": "unhandled",
"count": 3,
"data_type": "timedelta64[ns]",
"missing": 1,
"name": "duration",
"pos": 0,
}
json.dumps(actual, allow_nan=False)


def test_compute_stats_arrow_date_series_is_unhandled():
pa = pytest.importorskip("pyarrow")
series = pd.Series([date(2024, 1, 1), None], name="day", dtype=pd.ArrowDtype(pa.date32()))

actual = ps.compute_stats_series(series, "test", {})

assert actual["observability_value"]["day"]["base_data_type"] == "unhandled"
assert actual["observability_value"]["day"]["missing"] == 1
json.dumps(actual, allow_nan=False)
35 changes: 2 additions & 33 deletions ui/sdk/tests/tracking/test_polars_stats.py
Original file line number Diff line number Diff line change
Expand Up @@ -150,25 +150,12 @@ def test_compute_stats_df():
"pos": 6,
},
"h": {
"base_data_type": "datetime",
"base_data_type": "unhandled",
"count": 5,
"data_type": "Date",
"histogram": {},
"max": "2024-01-14",
"mean": "2024-01-06T19:12:00",
"min": "2024-01-01",
"missing": 0,
"name": "h",
"pos": 7,
"quantiles": {
0.1: "2024-01-01T00:00:00",
0.25: "2024-01-03T00:00:00",
0.5: "2024-01-06T00:00:00",
0.75: "2024-01-10T00:00:00",
0.9: "2024-01-14T00:00:00",
},
"std": 0.0,
"zeros": 0,
},
"i": {
"base_data_type": "numeric",
Expand All @@ -186,33 +173,15 @@ def test_compute_stats_df():
"zeros": 0,
},
"j": {
"base_data_type": "datetime",
"base_data_type": "unhandled",
"count": 5,
"data_type": "Date",
"histogram": {},
"max": "2022-05-01",
"mean": "2022-03-02T00:00:00",
"min": "2022-01-01",
"missing": 0,
"name": "j",
"pos": 9,
"quantiles": {
0.1: "2022-01-01T00:00:00",
0.25: "2022-02-01T00:00:00",
0.5: "2022-03-01T00:00:00",
0.75: "2022-04-01T00:00:00",
0.9: "2022-05-01T00:00:00",
},
"std": 0.0,
"zeros": 0,
},
},
}
# Date quantile behavior varies across polars versions — some return populated values,
# others return empty {}. Don't assert on quantiles for Date columns.
for col in ("h", "j"):
actual["observability_value"][col].pop("quantiles", None)
expected_stats["observability_value"][col].pop("quantiles", None)
assert actual == expected_stats


Expand Down
15 changes: 12 additions & 3 deletions ui/sdk/tests/tracking/test_runs.py
Original file line number Diff line number Diff line change
Expand Up @@ -510,8 +510,17 @@ def create_node(name: str, type_: type) -> node.Node:
(
pl.date_range(date(2023, 1, 1), date(2023, 1, 3), eager=True).alias("a"),
create_node("a", pl.Series),
"dagworks_describe", # TODO: fix quantiles
{"a": {"base_data_type": "unknown-polars", "data_type": "Date", "name": "a", "pos": 0}},
"dagworks_describe",
{
"a": {
"base_data_type": "unhandled",
"count": 3,
"data_type": "Date",
"missing": 0,
"name": "a",
"pos": 0,
}
},
),
],
ids=[
Expand All @@ -531,7 +540,7 @@ def create_node(name: str, type_: type) -> node.Node:
"polars_df",
"polars_series",
"pl_nan_series",
"pl_datetime_series",
"pl_date_series",
],
)
def test_process_result_happy(test_result, test_node, observability_type, stats):
Expand Down