diff --git a/hamilton/io/utils.py b/hamilton/io/utils.py index 020b8e4e1..dc10cc6db 100644 --- a/hamilton/io/utils.py +++ b/hamilton/io/utils.py @@ -22,7 +22,7 @@ from datetime import datetime from os import PathLike from pathlib import Path -from typing import Any, Literal +from typing import IO, Any, Literal from urllib import parse import pandas as pd @@ -66,7 +66,7 @@ def _is_query(query_or_table: str, operation: SqlOperation | None, legacy: bool) return not legacy and operation != "write" and _starts_with_select(query_or_table) -def get_file_metadata(path: str | Path | PathLike) -> dict[str, Any]: +def get_file_metadata(path: str | Path | PathLike | IO) -> dict[str, Any]: """Gives metadata from loading a file. Note: we reserve the right to change this schema. So if you're using this come @@ -77,19 +77,32 @@ def get_file_metadata(path: str | Path | PathLike) -> dict[str, Any]: - the file path - the last modified time - the current time + + ``path`` can also be a file object. A handle opened on a local file is described by its + file name. An in-memory buffer such as ``io.BytesIO`` has no file to describe, so its + path is its type name in angle brackets (e.g. ````) and its size is ``None``. """ - if isinstance(path, Path): - path = str(path) + in_memory = False + if not isinstance(path, (str, PathLike)): + name = getattr(path, "name", None) + if isinstance(name, (str, PathLike)): + path = name + else: + in_memory = True + path = f"<{type(path).__name__}>" + path = os.fspath(path) parsed = parse.urlparse(path) size = None scheme = parsed.scheme last_modified = time.time() timestamp = datetime.now().utcnow().timestamp() notes = f"File metadata is unsupported for scheme: {scheme} or path: {path} does not exist." + if in_memory: + notes = f"File metadata is unsupported for in-memory file object: {path}." # Check if the path is a local file path (Windows drive can be listed as a scheme) is_win_path = parsed.scheme and len(parsed.scheme) == 1 and parsed.scheme.isalpha() - if (parsed.scheme == "" or is_win_path) and os.path.exists(path): + if not in_memory and (parsed.scheme == "" or is_win_path) and os.path.exists(path): size = os.path.getsize(path) last_modified = os.path.getmtime(path) notes = "" diff --git a/hamilton/plugins/matplotlib_extensions.py b/hamilton/plugins/matplotlib_extensions.py index ccec535fa..f8a9e3f4f 100644 --- a/hamilton/plugins/matplotlib_extensions.py +++ b/hamilton/plugins/matplotlib_extensions.py @@ -84,7 +84,6 @@ def _get_saving_kwargs(self) -> dict: def save_data(self, data: Figure) -> dict[str, Any]: data.savefig(fname=self.path, **self._get_saving_kwargs()) - # TODO make utils.get_file_metadata() safer for when self.path is IO type return utils.get_file_metadata(self.path) @classmethod diff --git a/tests/io/test_utils.py b/tests/io/test_utils.py index 9dc487984..ac65fdb66 100644 --- a/tests/io/test_utils.py +++ b/tests/io/test_utils.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. +import io import json import pathlib import sqlite3 @@ -212,6 +213,24 @@ def test_get_file_metadata(tmp_path: pathlib.Path): assert metadata["file_metadata"]["timestamp"] is not None +def test_get_file_metadata_in_memory_file_object(): + metadata = get_file_metadata(io.BytesIO(b"test")) + assert metadata["file_metadata"]["path"] == "" + assert metadata["file_metadata"]["size"] is None + assert metadata["file_metadata"]["scheme"] == "" + assert metadata["file_metadata"]["notes"] != "" + + +def test_get_file_metadata_open_file_handle(tmp_path: pathlib.Path): + file_path = tmp_path / "test.txt" + file_path.write_text("test") + with open(file_path, "rb") as f: + metadata = get_file_metadata(f) + assert metadata["file_metadata"]["path"] == str(file_path) + assert metadata["file_metadata"]["size"] == 4 + assert metadata["file_metadata"]["notes"] == "" + + def test_get_file_metadata_url_schema(): url = "s3://bucket/key" metadata = get_file_metadata(url)