Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 18 additions & 5 deletions hamilton/io/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
from datetime import datetime
from os import PathLike
from pathlib import Path
from typing import Any, Literal
from typing import IO, Any, Literal
from urllib import parse

import pandas as pd
Expand Down Expand Up @@ -66,7 +66,7 @@ def _is_query(query_or_table: str, operation: SqlOperation | None, legacy: bool)
return not legacy and operation != "write" and _starts_with_select(query_or_table)


def get_file_metadata(path: str | Path | PathLike) -> dict[str, Any]:
def get_file_metadata(path: str | Path | PathLike | IO) -> dict[str, Any]:
"""Gives metadata from loading a file.

Note: we reserve the right to change this schema. So if you're using this come
Expand All @@ -77,19 +77,32 @@ def get_file_metadata(path: str | Path | PathLike) -> dict[str, Any]:
- the file path
- the last modified time
- the current time

``path`` can also be a file object. A handle opened on a local file is described by its
file name. An in-memory buffer such as ``io.BytesIO`` has no file to describe, so its
path is its type name in angle brackets (e.g. ``<BytesIO>``) and its size is ``None``.
"""
if isinstance(path, Path):
path = str(path)
in_memory = False
if not isinstance(path, (str, PathLike)):
name = getattr(path, "name", None)
if isinstance(name, (str, PathLike)):
path = name
else:
in_memory = True
path = f"<{type(path).__name__}>"
path = os.fspath(path)
parsed = parse.urlparse(path)
size = None
scheme = parsed.scheme
last_modified = time.time()
timestamp = datetime.now().utcnow().timestamp()
notes = f"File metadata is unsupported for scheme: {scheme} or path: {path} does not exist."
if in_memory:
notes = f"File metadata is unsupported for in-memory file object: {path}."

# Check if the path is a local file path (Windows drive can be listed as a scheme)
is_win_path = parsed.scheme and len(parsed.scheme) == 1 and parsed.scheme.isalpha()
if (parsed.scheme == "" or is_win_path) and os.path.exists(path):
if not in_memory and (parsed.scheme == "" or is_win_path) and os.path.exists(path):
size = os.path.getsize(path)
last_modified = os.path.getmtime(path)
notes = ""
Expand Down
1 change: 0 additions & 1 deletion hamilton/plugins/matplotlib_extensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -84,7 +84,6 @@ def _get_saving_kwargs(self) -> dict:

def save_data(self, data: Figure) -> dict[str, Any]:
data.savefig(fname=self.path, **self._get_saving_kwargs())
# TODO make utils.get_file_metadata() safer for when self.path is IO type
return utils.get_file_metadata(self.path)

@classmethod
Expand Down
19 changes: 19 additions & 0 deletions tests/io/test_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.

import io
import json
import pathlib
import sqlite3
Expand Down Expand Up @@ -212,6 +213,24 @@ def test_get_file_metadata(tmp_path: pathlib.Path):
assert metadata["file_metadata"]["timestamp"] is not None


def test_get_file_metadata_in_memory_file_object():
metadata = get_file_metadata(io.BytesIO(b"test"))
assert metadata["file_metadata"]["path"] == "<BytesIO>"
assert metadata["file_metadata"]["size"] is None
assert metadata["file_metadata"]["scheme"] == ""
assert metadata["file_metadata"]["notes"] != ""


def test_get_file_metadata_open_file_handle(tmp_path: pathlib.Path):
file_path = tmp_path / "test.txt"
file_path.write_text("test")
with open(file_path, "rb") as f:
metadata = get_file_metadata(f)
assert metadata["file_metadata"]["path"] == str(file_path)
assert metadata["file_metadata"]["size"] == 4
assert metadata["file_metadata"]["notes"] == ""


def test_get_file_metadata_url_schema():
url = "s3://bucket/key"
metadata = get_file_metadata(url)
Expand Down