diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index cf48d4b6..b89cc051 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -11,15 +11,7 @@ # pylint: disable=no-name-in-module from psycopg2.errors import UndefinedObject # ty: ignore[unresolved-import] -from sqlalchemy import ( - Column, - Connection, - Engine, - ForeignKey, - create_engine, - event, - select, -) +from sqlalchemy import Connection, Engine, ForeignKey, create_engine, event, select from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import ( IntegrityError, @@ -169,23 +161,32 @@ def create_db_engine( logger.error("DSN %s is malformed: %s", db_dsn, exc) raise Exit(1) from exc - settings = {} + settings: list[str] = [] + dialect_name = get_sync_engine(engine).dialect.name if schema_name is not None: - if get_sync_engine(engine).dialect.name == "mssql": - engine = engine.execution_options(schema_translate_map={None: schema_name}) - else: - settings["search_path"] = schema_name + match dialect_name: + case "mssql": + engine = engine.execution_options( + schema_translate_map={None: schema_name} + ) + case "snowflake": + settings.append(f"USE SCHEMA {schema_name};") + case _: + settings.append(f"SET search_path TO {schema_name};") dataset_views: list[str] = [] if parquet_dir is not None: - tables = find_parquet_tables(parquet_dir) - joined = ",".join( - sorted({str(t.root.parent) for t in tables if not t.is_dataset}) - ) - # double up single quotes - dj = joined.replace("'", "''") - # enclose in single quotes - settings["file_search_path"] = f"'{dj}'" - dataset_views = [_dataset_view_sql(t) for t in tables if t.is_dataset] + match dialect_name: + case "duckdb": + tables = find_parquet_tables(parquet_dir) + # double up single quotes + dj = str(parquet_dir).replace("'", "''") + settings.append(f"SET file_search_path TO '{dj}';") + dataset_views = [_dataset_view_sql(t) for t in tables if t.hive_files()] + case _: + tables = [] + logger.error( + "--parquet-dir cannot be used with %s, only duckdb.", dialect_name + ) if settings or dataset_views: event_engine = get_sync_engine(engine) @@ -227,11 +228,15 @@ def create_db_engine_dst( return create_db_engine(db_dsn, schema_name, use_asyncio) -def get_metadata(engine: Engine, schema_name: Optional[str] = None) -> MetaData: +def get_metadata( + engine: Engine, + schema_name: Optional[str], + views_to_tables: bool, +) -> MetaData: """Get the MetaData object associated with the engine passed.""" md = MetaData() try: - md.reflect(engine, schema=schema_name) + md.reflect(engine, schema=schema_name, views=views_to_tables) except OperationalError as exc: logger.error("Cannot connect to database: %s", exc) raise Exit(1) from exc @@ -245,7 +250,7 @@ def _sql_quote(text: str, quote: str) -> str: def _dataset_view_sql(table: ParquetTable) -> str: """Make the SQL that exposes a directory of parquet files as one table.""" - files = ",".join(_sql_quote(str(f), "'") for f in table.files) + files = ",".join(_sql_quote(str(f), "'") for f in table.hive_files()) return ( f"CREATE OR REPLACE TEMP VIEW {_sql_quote(table.name, chr(34))} AS " f"SELECT * FROM read_parquet([{files}], " @@ -267,16 +272,14 @@ def create_dataset_views( cursor.close() -def set_db_settings(connection: DBAPIConnection, settings: Mapping[str, str]) -> None: - """Set the SEARCH_PATH for a PostgreSQL connection.""" +def set_db_settings(connection: DBAPIConnection, settings: list[str]) -> None: + """Run commands.""" # https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#remote-schema-table-introspection-and-postgresql-search-path existing_autocommit = connection.autocommit connection.autocommit = True cursor = connection.cursor() - # Parametrised queries don't work with asyncpg, hence the f-string. - sql = "".join(f"SET {k} TO {v};" for k, v in settings.items()) - cursor.execute(sql) + cursor.execute("\n".join(settings)) cursor.close() connection.autocommit = existing_autocommit @@ -299,25 +302,6 @@ def get_orm_metadata( return metadata -def get_fk_column_between( - from_: Table, to_: Table -) -> tuple[Column, ForeignKey] | tuple[None, None]: - """ - Get a foreign key between two tables. - - :param from_: The table on which the foreign key should be defined. - :param to_: The table to which the foreign key should refer. - :return: A pair (column on from_, foreign key to to_) if such a key exists, - or (None, None) if not. - """ - for c in from_.columns: - fk: ForeignKey - for fk in c.foreign_keys: - if fk.column.table == to_: - return (c, fk) - return (None, None) - - def is_numeric_sql_type(column_type: TypeEngine | None) -> bool: """Whether a SQLAlchemy column type is Numeric or Integer.""" return isinstance(column_type, (Numeric, Integer)) diff --git a/datafaker/dialects.py b/datafaker/dialects.py index 9d92e7b3..2b0e98c6 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -3,6 +3,7 @@ from collections.abc import Mapping from typing import Any, Optional, TypeVar +import snowflake.sqlalchemy.custom_types as snowflake_types from sqlalchemy import Column, Select, Table from sqlalchemy.ext.compiler import compiles from sqlalchemy.schema import CreateSchema, CreateTable @@ -515,3 +516,19 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str: same random number gets chosen for each row. """ return "NEWID()" + + +@compiles(snowflake_types.TIMESTAMP_LTZ) +def compile_timestampltz_generic( + _element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any +) -> str: + """Render TIMESTAMP_LTZ as DateTime.""" + return compiler.process(DateTime(timezone=False), **kw) + + +@compiles(snowflake_types.TIMESTAMP_LTZ, "snowflake") +def compile_timestampltz_ddb( + element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any +) -> str: + """Render TIMESTAMP_LTZ as TIMESTAMP_LTZ.""" + return compiler.process(element, **kw) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 5d6dd0a8..cde6d532 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Callable, Optional, cast -from sqlalchemy import Column, and_, literal_column, select +from sqlalchemy import Column, and_, select from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private @@ -38,6 +38,7 @@ schema_qualified_name, set_property, split_column_full_name, + text_list, ) @@ -161,7 +162,8 @@ def do(self, column_roles: RoleEntry | None) -> RoleEntry | None: """Perform the command.""" if column_roles is None: return None - column_roles.new.remove(self.role) + if self.role in column_roles.new: + column_roles.new.remove(self.role) return column_roles @@ -198,6 +200,9 @@ class GeneratorCmd(DbCmd): PRIMARY_PRIVATE_TEXT = "Primary Private" SECONDARY_PRIVATE_TEXT = "Secondary Private on columns {0}" NOT_PRIVATE_TEXT = "Not private" + REQUIRES_NO_SOURCE_DATA_TEXT = ( + "{0}. {2}{1}{3} requires no data from the source database." + ) REQUIRES_SOURCE_DATA_TEXT = ( "{0}. {2}{1}{3} requires the following data from the source database:" ) @@ -472,7 +477,7 @@ def _copy_roles(self) -> None: """Set role information in the configuration.""" for table_name, table_roles in self.roles.items(): for column_name, entry in table_roles.items(): - if entry.new: + if entry.new or entry.old: set_property( self.config, ["tables", table_name, "columns", column_name, "roles"], @@ -534,8 +539,8 @@ def _print_role_changes(self) -> int: "Changing role set of column {0} of table {1} from {2} to {3}", column_name, table_name, - ", ".join(str(oe) for oe in entry.old), - ", ".join(str(ne) for ne in entry.new), + text_list((e.value for e in entry.old), "empty"), + text_list((e.value for e in entry.new), "empty"), ) return count @@ -904,7 +909,7 @@ def _print_values_queried(self, table_name: str, n: int, prop: Proposer) -> None theme = get_active_theme() if not prop.select_aggregate_clauses() and not prop.custom_queries(): self.print( - "{0}. {2}{1}{3} requires no data from the source database.", + self.REQUIRES_NO_SOURCE_DATA_TEXT, n, prop.name(), theme.function, @@ -1027,12 +1032,13 @@ def _print_select_aggregate_query(self, table_name: str, prop: Proposer) -> None def _get_column_data( self, count: int, to_str: Callable[[Any], str] = repr ) -> list[list[str]]: - columns = self._get_column_names() - col_exprs = [literal_column(col) for col in columns] - nonnull_clauses = [literal_column(col).isnot(None) for col in columns] + column_names = self._get_column_names() + table = self.table_metadata() + columns = [table.columns[c] for c in column_names] + nonnull_clauses = [col.isnot(None) for col in columns] stmt = ( - select(*col_exprs) - .select_from(self.table_metadata()) + select(*columns) + .select_from(table) .where(and_(*nonnull_clauses)) .order_by(Random()) .limit(count) diff --git a/datafaker/interactive/missingness.py b/datafaker/interactive/missingness.py index 60940a36..bc41557c 100644 --- a/datafaker/interactive/missingness.py +++ b/datafaker/interactive/missingness.py @@ -215,7 +215,7 @@ def _copy_entries(self) -> None: "kwargs": { "patterns": f'SRC_STATS["{src_stat_key}"]["results"]' }, - "columns": [c.name for c in entry.new_type.columns], + "columns": [str(c.name) for c in entry.new_type.columns], } ] src_stats.append( diff --git a/datafaker/interactive/table.py b/datafaker/interactive/table.py index 7a2852c0..107a53db 100644 --- a/datafaker/interactive/table.py +++ b/datafaker/interactive/table.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any, cast -from sqlalchemy import func, literal_column, select +from sqlalchemy import func, select from datafaker.dialects import Random from datafaker.interactive.base import ( @@ -469,26 +469,21 @@ def complete_data( return [] return self.get_column_completions(text) - def print_column_data(self, column: str, count: int, min_length: int) -> None: + def print_column_data(self, column_name: str, count: int, min_length: int) -> None: """ Print a sample of data from a certain column of the current table. - :param column: The name of the column to report on. + :param column_name: The name of the column to report on. :param count: The number of rows to sample. :param min_length: The minimum length of text to choose from (0 for any text). """ - col_expr = literal_column(column) + table = self.table_metadata() + column = table.columns[column_name] if 0 < min_length: - where_clause = func.length(col_expr) >= min_length + where_clause = func.length(column) >= min_length else: - where_clause = col_expr.isnot(None) - stmt = ( - select(col_expr) - .select_from(self.table_metadata()) - .where(where_clause) - .order_by(Random()) - .limit(count) - ) + where_clause = column.isnot(None) + stmt = select(column).where(where_clause).order_by(Random()).limit(count) with self.sync_engine.connect() as connection: result = connection.execute(stmt) self.columnize([str(x[0]) for x in result.all()]) diff --git a/datafaker/main.py b/datafaker/main.py index b180b97b..4a94fde9 100644 --- a/datafaker/main.py +++ b/datafaker/main.py @@ -172,11 +172,17 @@ def make_tables( help=( "Directory of Parquet files to consider part of the database." " This can be useful when using DuckDB." + " The .parquet files or directories containing partitioned tables" + " must all be directly inside this named directory." " Make sure you check the output!" ), file_okay=False, dir_okay=True, ), + views_to_tables: bool = Option( + False, + help=("Generate all views as tables in the synthetic version."), + ), ) -> None: """Make a YAML file representing the tables in the schema. @@ -192,7 +198,9 @@ def make_tables( content = make_tables_file( get_source_dsn(), get_source_schema(), - parquet_dir, + views_to_tables=views_to_tables, + parquet_dir=parquet_dir, + guess_keys=views_to_tables or parquet_dir is not None, ) orm_file_path.write_text(content, encoding="utf-8") logger.debug("%s created.", orm_file) diff --git a/datafaker/make.py b/datafaker/make.py index d9ddf25b..d1a4f842 100644 --- a/datafaker/make.py +++ b/datafaker/make.py @@ -10,6 +10,7 @@ from typing import Any, Final, Optional, Tuple, Type import pandas as pd +import snowflake.sqlalchemy.custom_types as snowflake_types import snsql import typer import yaml @@ -39,7 +40,7 @@ get_sync_engine, get_vocabulary_table_names, ) -from datafaker.parquet2orm import get_parquet_orm +from datafaker.parquet2orm import add_guessed_keys, get_initial_parquet_orm from datafaker.settings import get_source_dsn, get_source_schema from datafaker.utils import ( get_columns_assigned, @@ -58,6 +59,8 @@ if issubclass(entry, BaseProvider) and entry.__module__ == "datafaker.providers": PROVIDER_IMPORTS.append(entry_name) +DEFAULT_UNIQUE_CONSTRAINT_TRIES: int | None = 500 + @dataclass class VocabularyTableGeneratorInfo: @@ -378,6 +381,10 @@ def get_result_mappings( generator=_string_generator, choice=True, ), + snowflake_types.TIMESTAMP_LTZ: GeneratorInfo( + generator="generic.datetime.datetime", + arg_types={"start": int, "end": int}, + ), } @@ -654,6 +661,8 @@ def get_generation_info( max_unique_constraint_tries = get_property_or_none( config, "max-unique-constraint-tries", int ) + if max_unique_constraint_tries is None: + max_unique_constraint_tries = DEFAULT_UNIQUE_CONSTRAINT_TRIES return GenerationInfo( provider_imports=PROVIDER_IMPORTS, row_generator_module_name=row_generator_module_name, @@ -694,26 +703,34 @@ def _generate_vocabulary_table( download_table(table, engine, yaml_file_name, compress) +# pylint: disable=too-many-arguments disable=too-many-positional-arguments def make_tables_file( db_dsn: str, schema_name: Optional[str], + views_to_tables: bool, parquet_dir: Optional[Path] = None, engine: Optional[Engine] = None, + guess_keys: bool = False, ) -> str: """Construct the YAML file representing the schema.""" if engine is None: engine = get_sync_engine(create_db_engine(db_dsn, schema_name=schema_name)) - metadata = get_metadata(engine, schema_name=schema_name) + metadata = get_metadata( + engine, schema_name=schema_name, views_to_tables=views_to_tables + ) meta_dict = metadata_to_dict(metadata, schema_name, engine, parquet_dir) if parquet_dir is not None: - extra_meta = get_parquet_orm(parquet_dir) + extra_meta = get_initial_parquet_orm(parquet_dir) if extra_meta: md_tables: dict[str, Any] = get_property(meta_dict, "tables", {}) new_tables = {**extra_meta, **md_tables} meta_dict["tables"] = new_tables + if guess_keys: + add_guessed_keys(meta_dict["tables"]) + return yaml.dump(meta_dict, sort_keys=False) diff --git a/datafaker/parquet2orm.py b/datafaker/parquet2orm.py index a8d5030e..f801ceb3 100644 --- a/datafaker/parquet2orm.py +++ b/datafaker/parquet2orm.py @@ -1,15 +1,15 @@ """Add to ORM structure based on parquet files.""" +import glob import re -from collections.abc import Iterable, Mapping -from dataclasses import dataclass +from abc import ABC +from collections.abc import Generator, Mapping from pathlib import Path from typing import Any import numpy as np -from fastparquet import ParquetFile -from typer import Exit +from fastparquet import ParquetException, ParquetFile -from datafaker.utils import logger +from datafaker.utils import get_property, logger PARQUET_SUFFIXES = {".parquet", ".parq"} @@ -17,35 +17,36 @@ # Partition keys are not stored in the files themselves. PARTITION_KEY_DTYPE = "category" -_hive_dir_re = re.compile(r"^([^=]+)=(.*)$") +KEY_TYPES = frozenset(["INTEGER", "UUID"]) -@dataclass(frozen=True) -class ParquetTable: - """A table found in a directory of parquet files.""" +class ParquetTable(ABC): + """A table found in a parquet file or directory of parquet files.""" - name: str - """The name of the table, as it will appear in ``orm.yaml``.""" - root: Path - """The parquet file for a single-file table, or the directory of a dataset.""" - files: tuple[Path, ...] - """All the parquet files that make up the table, sorted.""" - is_dataset: bool - """True if the table is a (possibly partitioned) directory of parquet files.""" + def __init__(self, pf: ParquetFile, name: str) -> None: + """Initialize a ParquetTable.""" + self.parquet_file = pf + self.name = name + def hive_files(self) -> list[Path]: + """Get any files needing to be merged into a view.""" + return [] -def _parquet_files_under(directory: Path) -> tuple[Path, ...]: - """Find all the parquet files in ``directory`` and below, sorted.""" - return tuple( - sorted( - entry - for entry in directory.rglob("*") - if entry.suffix in PARQUET_SUFFIXES and entry.is_file() - ) - ) +class ParquetHiveTable(ParquetTable): + """A hive partitioned table formed from a directory.""" -def find_parquet_tables(directory: Path | str) -> list[ParquetTable]: + def __init__(self, pf: ParquetFile, name: str, file_list: list[Path]): + """Initialize a ParquetHiveTable.""" + super().__init__(pf, name) + self.file_list = file_list + + def hive_files(self) -> list[Path]: + """Return the list of files that make up this hive.""" + return self.file_list + + +def find_parquet_tables(directory: Path | str) -> Generator[ParquetTable, None, None]: """ Find the tables represented by the parquet files in a directory. @@ -57,129 +58,67 @@ def find_parquet_tables(directory: Path | str) -> list[ParquetTable]: (named ``key=value``), then ``directory`` is itself a single table. :param directory: The directory to search for parquet files in. - :return: The tables found, files first, then datasets, each sorted by name. + :return: The ParquetFiles found, sorted by name. """ directory = Path(directory) - children = sorted(directory.iterdir()) - file_tables = [ - ParquetTable(entry.name, entry, (entry,), False) - for entry in children - if entry.suffix in PARQUET_SUFFIXES and entry.is_file() - ] - dir_files = [ - (entry, files) - for entry in children - if entry.is_dir() - for files in [_parquet_files_under(entry)] - if files - ] - if not file_tables and dir_files: - if all(_hive_dir_re.match(entry.name) for entry, _ in dir_files): - name = directory.resolve().name - return [ - ParquetTable( - name, - directory, - tuple(f for _, files in dir_files for f in files), - True, - ) - ] - return file_tables + [ - ParquetTable(entry.name, entry, files, True) for entry, files in dir_files - ] - - -def _file_dtypes(path: Path) -> dict[str, Any]: - """Get the column names and dtypes stored in one parquet file.""" - return dict(ParquetFile(path).dtypes) - - -def _partition_keys(table: ParquetTable, path: Path) -> list[str]: - """Get the Hive partition keys of one file of a dataset, in path order.""" - keys = [] - for part in path.relative_to(table.root).parts[:-1]: - match = _hive_dir_re.match(part) - if match: - keys.append(match.group(1)) - return keys - - -def _describe_column_difference( - expected: Mapping[str, Any], actual: Mapping[str, Any] -) -> str: - """Describe how the column names of ``actual`` differ from ``expected``.""" - problems = [] - missing = [c for c in expected if c not in actual] - extra = [c for c in actual if c not in expected] - if missing: - problems.append(f"missing columns {missing}") - if extra: - problems.append(f"extra columns {extra}") - return "; ".join(problems) - - -def _type_name(dtype: Any) -> str: - """Get a name for a dtype that ignores case, so ``int32`` equals ``Int32``.""" - return str(dtype).lower() - - -def get_table_dtypes(table: ParquetTable) -> dict[str, Any]: + for child in sorted(directory.iterdir()): + try: + if child.is_dir(): + files = glob.glob("**/*.parquet", root_dir=child, recursive=True) + abs_files = [child / f for f in files] + pf = ParquetFile(abs_files, root=child, verify=True) + yield ParquetHiveTable(pf, child.name, abs_files) + else: + pf = ParquetFile(child, verify=True) + yield ParquetTable(pf, child.name) + except ParquetException as e: + logger.error("Entry %s cannot be read as parquet", child.name) + logger.warning(e) + except ValueError as e: + logger.error("Entry %s cannot be read as parquet", child.name) + logger.warning(e) + except IndexError as _e: + logger.warning("Entry %s is empty", child.name) + + +def add_guessed_keys(tables_conf: dict[str, Any]): """ - Get the columns and dtypes of a table, checking that its files agree. - - All the files of a partitioned table must have the same column names and - the same partition keys. Directories that contain unrelated tables are - rejected with an error message. Column types may differ between files - (for example when a partition has no values for a column); this is - reported as a warning and the type from the first file is used. + Read the orm.yaml config "tables" portion, adding guesses for foreign keys and primary keys. - :param table: The table to examine. - :return: A dict of column names to dtypes. Partition keys are included. - :raises typer.Exit: If the files of a dataset do not all have the same columns. + :param tables_conf: On input, the tables portion of the configuration file; this dictionary will + be changed to set the appropriate primary and foreign keys. """ - first = table.files[0] - dtypes = _file_dtypes(first) - if not table.is_dataset: - return dtypes - keys = _partition_keys(table, first) - type_names: dict[str, set[str]] = {} - for path in table.files[1:]: - file_dtypes = _file_dtypes(path) - problem = _describe_column_difference(dtypes, file_dtypes) - if not problem and _partition_keys(table, path) != keys: - problem = "different partition directories" - if problem: - logger.error( - "Directory %s cannot be read as a single partitioned table: " - "%s differs from %s (%s). All the files in a partitioned " - "table must have the same columns. If the " - "directories inside %s hold different tables, move each " - "table into its own directory directly inside the parquet " - "directory and run this command again.", - table.name, - path, - first, - problem, - table.root, + guesser = _ColumnGuesser(tables_conf) + for table_name, table_conf in tables_conf.items(): + columns = get_property(table_conf, "columns", {}) + table_conf["columns"] = columns + primaries = set() + for column_name, column_conf in columns.items(): + ctype = get_property(column_conf, "type", "INTEGER") + if ctype in KEY_TYPES: + if guesser.is_likely_primary(table_name, column_name): + primaries.add(column_name) + column_conf["primary"] = True + column_conf["nullable"] = False + elif guess := guesser.get_likely_foreign_key_target(column_name): + column_conf["foreign_keys"] = [f"{guess[0]}.{guess[1]}"] + if column_name == "id": + column_conf["primary"] = True + column_conf["nullable"] = False + if len(primaries) == 0: + logger.warning( + "Found no likely primary keys for table %s", + table_name, + ) + elif 1 < len(primaries): + logger.warning( + "Found multiple likely primary keys for table %s: %s", + table_name, + ", ".join(sorted(list(primaries))), ) - raise Exit(1) - for column, dtype in file_dtypes.items(): - if _type_name(dtype) != _type_name(dtypes[column]): - names = type_names.setdefault(column, {_type_name(dtypes[column])}) - names.add(_type_name(dtype)) - for column, names in type_names.items(): - logger.warning( - "Column %s.%s has different types in different files: %s. " - "Using the type from %s.", - table.name, - column, - sorted(names), - first, - ) - return {**dtypes, **{k: PARTITION_KEY_DTYPE for k in keys if k not in dtypes}} - - -def get_parquet_orm(directory: Path) -> dict[str, Any] | None: + + +def get_initial_parquet_orm(directory: Path) -> dict[str, Any] | None: """ Read the parquet files, guess a database structure from that. @@ -193,13 +132,8 @@ def get_parquet_orm(directory: Path) -> dict[str, Any] | None: if not directory.is_dir(): logger.error("%s is not a directory", directory) return None - tables = find_parquet_tables(directory) - dtypes = {table.name: get_table_dtypes(table) for table in tables} - guesser = _ColumnGuesser(dtypes) - return { - name: _get_table_orm(table_dtypes, name, guesser) - for name, table_dtypes in dtypes.items() - } + parquets = find_parquet_tables(directory) + return {p.name: _get_table_orm(p.parquet_file) for p in parquets} _camel_case_re = re.compile(r"([a-z])([A-Z])") @@ -215,14 +149,21 @@ def _get_words(s: str) -> set[str]: class _ColumnGuesser: """Guesses foreign key targets.""" - def __init__(self, tables: Mapping[str, Iterable[str]]) -> None: - """Initialize the column guesser from table names to column names.""" + def __init__(self, tables: Mapping[str, Any]) -> None: + """ + Initialize the column guesser from table names to column names. + + :param tables: The "tables" key from the orm.yaml structure. + """ + # List of triples (table_name, column_name, list_of_words_in_table_and_column_name) self.tn_cn_words: list[tuple[str, str, set[str]]] = [] + # Dict from table name to the words in that table name self.tn2words: dict[str, set[str]] = {} - for name, table in tables.items(): + for name, table_conf in tables.items(): table_words = _get_words(name) self.tn2words[name] = table_words - for column in table: + columns = get_property(table_conf, "columns", {}) + for column in columns.keys(): self.tn_cn_words.append( (name, column, table_words | _get_words(column)) ) @@ -235,7 +176,8 @@ def get_likely_foreign_key_target( Get a foreign key target string if the name suggests it might be. :param column_name: The name of the column which might be a foreign key - :return: "table.column" or None if no column seems like a likely target + :return: A pair (table_name, column_name) or None if no column seems + like a likely target. """ our_words = _get_words(column_name) max_overlaps_so_far = (1, len(our_words) - 1, 0) @@ -256,64 +198,25 @@ def get_likely_foreign_key_target( best_so_far = (tn, cn) return best_so_far + def is_likely_primary(self, table_name, column_name) -> bool: + """Determine if the specified column name looks like a primary key.""" + words = _get_words(column_name) - {"id", "key"} + return words <= self.tn2words[table_name] -def _get_table_orm( - dtypes: Mapping[str, Any], name: str, column_guesser: _ColumnGuesser -) -> dict[str, Any]: + +def _get_table_orm(parquet_file: ParquetFile) -> dict[str, Any]: """ Guess the ORM configuration of the table passed in. - :param dtypes: The column names and dtypes of the table to guess the config of. - :param name: The filename of the parquet file, or directory name of the dataset. - :param column_guesser: Guesses foreign key targets across all the tables. + :param dtypes: The parquet file to make the table configuration for. :return: A reasonable ORM configuration for this table. """ - column_types = {column: _dtype_to_sql(dtype) for column, dtype in dtypes.items()} - # Get everything before the last dot - # (or the whole thing if there is no dot) - name_pref = name.rsplit(".", 1)[0] - name_words = _get_words(name_pref) - cols_orm = {} - likely_primaries = [] - for column, ctype in column_types.items(): - # A primary key is likely to be the column name plus "_id" or something - words = _get_words(column) - {"id", "key"} - col_orm: dict[str, Any] = {} - if ctype == "INTEGER": - if words <= name_words: - likely_primaries.append(column) - col_orm["primary"] = True - col_orm["nullable"] = False - else: - col_orm["nullable"] = True - fk_pair = column_guesser.get_likely_foreign_key_target(column) - if fk_pair is not None: - fk_table, fk_column = fk_pair - if fk_table != name: - logger.debug( - "Column %s.%s guessed as being a foreign key to %s.%s", - name_pref, - column, - fk_table, - fk_column, - ) - col_orm["foreign_keys"] = [f"{fk_table}.{fk_column}"] - if ctype is not None: - logger.debug("Column %s.%s type guessed as %s", name_pref, column, ctype) - col_orm["type"] = ctype - else: - logger.warning( - "Could not determine type of column %s.%s", name_pref, column - ) - cols_orm[column] = col_orm - if len(likely_primaries) == 0: - logger.warning("No likely primary keys found for table %s", name) - elif 1 < len(likely_primaries): - logger.warning( - "Found multiple likely primary keys for table %s: %s", - name, - likely_primaries, - ) + cols_orm = { + column: { + "type": _dtype_to_sql(dtype), + } + for column, dtype in parquet_file.dtypes.items() + } return { "columns": cols_orm, "unique": [], diff --git a/datafaker/proposers/__init__.py b/datafaker/proposers/__init__.py index bee4c536..98c969a6 100644 --- a/datafaker/proposers/__init__.py +++ b/datafaker/proposers/__init__.py @@ -17,7 +17,7 @@ MultivariateNormalProposerFactory, ) from datafaker.proposers.extract import DateComponentExtractProposerFactory -from datafaker.proposers.intervals import DateAfterProposerFactory +from datafaker.proposers.intervals import CopyProposerFactory, DateAfterProposerFactory from datafaker.proposers.mimesis import ( MimesisDateProposerFactory, MimesisDateTimeProposerFactory, @@ -57,6 +57,7 @@ def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: NullPartitionedNormalProposerFactory(config, metadata), NullPartitionedLogNormalProposerFactory(config, metadata), DateAfterProposerFactory(config, metadata), + CopyProposerFactory(config, metadata), DateComponentExtractProposerFactory(config, metadata), IncrementProposerFactory(), ) diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 9e677326..6b456ca0 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -2,19 +2,21 @@ import re from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from typing import Any, Union import mimesis import mimesis.locales from sqlalchemy import Column, Engine, Join, Table, func, select from sqlalchemy.exc import DatabaseError +from sqlalchemy.orm import aliased from sqlalchemy.types import Integer, Numeric, String, TypeEngine from typing_extensions import Self from datafaker.dialects import StdDev from datafaker.providers import DistributionProvider -from datafaker.utils import logger +from datafaker.utils import T, logger NumericType = Union[int, float] @@ -306,6 +308,93 @@ def fit_from_buckets(xs: Sequence[NumericType], ys: Sequence[NumericType]) -> fl return sum_diff_squared / (count * count) +class ForeignKeyRelationship: + """How a table is related.""" + + def __init__(self, fk_column: Column, target_column: Column) -> None: + """ + Initialise a ``ForeignKeyRelationship``. + + :param fk_column: The column containing the foreign key (the source). + :param target_column: The column that ``fk_column`` matches in + the target table. + """ + self.fk_column = fk_column + table = aliased(target_column.table) + self.target_column = table.columns[target_column.name] + + def __repr__(self): + """Render the relationship as a JOIN.""" + return ( + f"" + ) + + def target_table(self) -> Table: + """Get the related table.""" + return self.target_column.table + + def target_table_name(self) -> str: + """ + Get the name of the target table. + + Don't use ``target_table().name`` because this will give the + name of the aliased table. + """ + return str(self.target_column.table.original.name) + + +def _get_one(seq: Iterable[T]) -> T | None: + """Get an arbitrary member of the sequence, or None if it is empty.""" + for elt in seq: + return elt + return None + + +def make_foreign_key_relationship(fk_column: Column) -> ForeignKeyRelationship | None: + """Make a ``ForeignKeyRelationship`` from a foreign key column.""" + if (elt := _get_one(fk_column.foreign_keys)) is None: + return None + return ForeignKeyRelationship(fk_column, elt.column) + + +@dataclass +class RelatedColumn: + """A column in a related table, and how it is related.""" + + target: Column + relationship: ForeignKeyRelationship | None + + def add_from_to_query(self, query: Any) -> Any: + """Take a query and add a ``select_from`` and maybe ``join`` as appropriate.""" + if self.relationship is None: + return query.select_from(self.target.table) + query = query.select_from(self.relationship.fk_column.table) + return self.add_join_to_query(query) + + def add_join_to_query(self, query: Any) -> Any: + """Take a query and maybe add ``join`` as appropriate.""" + rel = self.relationship + if rel is None: + return query + return query.join( + rel.target_column.table, + onclause=rel.fk_column == rel.target_column, + ) + + def target_table(self) -> Table | None: + """Get the related table, or None if it's the same table.""" + if self.relationship is None: + return None + return self.relationship.target_table() + + def target_table_name(self) -> str | None: + """Get the name of the target table, or None if it's the same table.""" + if self.relationship is None: + return None + return self.relationship.target_table_name() + + class Buckets: """ Measured buckets for a real distribution. @@ -323,12 +412,12 @@ def __init__( mean: float, stddev: float, count: int, - join_tables: list[Table] | None = None, + anchor_relation: RelatedColumn | None = None, ): """Initialise a Buckets object.""" self.mean = mean self.stddev = stddev - query = self._get_bucket_query(table, column, join_tables) + query = self._get_bucket_query(table, column, anchor_relation) with engine.connect() as connection: raw_buckets = connection.execute(query.group_by("b")) self.buckets: Sequence[int] = [0] * 10 @@ -350,7 +439,7 @@ def _get_bucket_query( self, table: Table | Join, column: Any, - join_tables: list[Table] | None, + anchor_relation: RelatedColumn | None = None, ) -> Any: """Make the query for half-sd buckets.""" bottom = self.mean - 2 * self.stddev @@ -359,8 +448,8 @@ def _get_bucket_query( func.count(column).label("f"), # pylint: disable=not-callable ((func.floor(column) - bottom) / width).label("b"), ).select_from(table) - for jt in join_tables or []: - query = query.join(jt) + if anchor_relation is not None: + query = anchor_relation.add_join_to_query(query) return query @classmethod @@ -369,7 +458,7 @@ def make_buckets( engine: Engine, table: Table | Join, column: Any, - join_tables: list[Table] | None = None, + anchor_relation: RelatedColumn | None = None, ) -> Self | None: """ Construct a Buckets object. @@ -383,14 +472,16 @@ def make_buckets( :param engine: SQLAlchemy engine. :param table: SQLAlchemy table (or joined tables) to pull data from. :param column: SQLAlchemy column or expression to measure. + :param anchor_relation: A relationship to another column whose table + needs to be joined. """ query = select( func.avg(column).label("mean"), StdDev(column).label("stddev"), func.count(column).label("count"), # pylint: disable=not-callable ).select_from(table) - for jt in join_tables or []: - query = query.join(jt) + if anchor_relation is not None: + query = anchor_relation.add_join_to_query(query) with engine.connect() as connection: result = connection.execute(query).first() if result is None or result.stddev is None or getattr(result, "count") < 2: @@ -403,7 +494,7 @@ def make_buckets( result.mean, result.stddev, getattr(result, "count"), - join_tables, + anchor_relation, ) except DatabaseError as exc: logger.debug("Failed to instantiate Buckets object: %s", exc) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index acf0e1f4..38e475f2 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -6,17 +6,7 @@ from abc import abstractmethod from typing import Any, Sequence, Union -from sqlalchemy import ( - Column, - CursorResult, - Engine, - desc, - func, - literal_column, - select, - table, - text, -) +from sqlalchemy import Column, CursorResult, Engine, desc, func, select from datafaker.dialects import Random from datafaker.proposers.base import ( @@ -25,7 +15,6 @@ dist_gen, fit_from_buckets, ) -from datafaker.utils import schema_qualified_name NumericType = Union[int, float] @@ -57,12 +46,10 @@ def zipf_distribution(total: int, bins: int) -> typing.Generator[int, None, None def _choice_stmt( # pylint: disable=R0913,R0917 - column_name: str, - table_name: str, + column: Column, store_counts: bool, sample_count: int | None, suppress_count: int, - table_sql: str | None = None, ) -> Any: """Build a SQLAlchemy SELECT for gathering choice value distributions. @@ -70,13 +57,10 @@ def _choice_stmt( # pylint: disable=R0913,R0917 TOP/newid() on MS-SQL. MS-SQL also forbids ORDER BY inside a subquery without TOP; this function never emits such a clause. """ - col = literal_column(f'"{column_name}"') - tbl = text(table_sql) if table_sql else table(table_name) if sample_count is not None: sample_sub = ( - select(col.label("value")) - .where(col.isnot(None)) - .select_from(tbl) + select(column.label("value")) + .where(column.isnot(None)) .order_by(Random()) .limit(sample_count) .subquery("_inner") @@ -92,12 +76,11 @@ def _choice_stmt( # pylint: disable=R0913,R0917 else: counted_sub = ( select( - col.label("value"), - func.count(col).label("count"), # pylint: disable=E1102 + column.label("value"), + func.count(column).label("count"), # pylint: disable=E1102 ) - .where(col.isnot(None)) - .select_from(tbl) - .group_by(col) + .where(column.isnot(None)) + .group_by(column) .subquery("_counted") ) out_cols = [counted_sub.c.value] @@ -119,31 +102,26 @@ class ChoiceProposer(Proposer): # pylint: disable=too-many-arguments too-many-positional-arguments def __init__( self, - table_name: str, - column_name: str, + column: Column, values: list[Any], counts: list[int], sample_count: int | None = None, suppress_count: int = 0, dialect: Any = None, - table_sql: str | None = None, ) -> None: """Initialise a ChoiceProposer.""" super().__init__() - self.table_name = table_name - self.column_name = column_name + self.column = column self.values = values estimated_counts = self.get_estimated_counts(counts) self._fit = fit_from_buckets(counts, estimated_counts) extra_comment = " and their counts" if self.STORE_COUNTS else "" stmt = _choice_stmt( - column_name, - table_name, + column, self.STORE_COUNTS, sample_count, suppress_count, - table_sql=table_sql, ) compile_opts: dict[str, Any] = {"compile_kwargs": {"literal_binds": True}} if dialect is not None: @@ -153,28 +131,28 @@ def __init__( if suppress_count == 0: if sample_count is None: self._comment = ( - f"All the values{extra_comment} that appear in column {column_name}" - f" of table {table_name}" + f"All the values{extra_comment} that appear in column {column.name}" + f" of table {column.table.name}" ) self._annotation = None else: self._comment = ( - f"The values{extra_comment} that appear in column {column_name}" - f" of a random sample of {sample_count} rows of table {table_name}" + f"The values{extra_comment} that appear in column {column.name}" + f" of a random sample of {sample_count} rows of table {column.table.name}" ) self._annotation = "sampled" else: if sample_count is None: self._comment = ( - f"All the values{extra_comment} that appear in column {column_name}" - f" of table {table_name} more than {suppress_count} times" + f"All the values{extra_comment} that appear in column {column.name}" + f" of table {column.table.name} more than {suppress_count} times" ) self._annotation = "suppressed" else: self._comment = ( f"The values{extra_comment} that appear more than {suppress_count} times" - f" in column {column_name}, out of a random sample of {sample_count} rows" - f" of table {table_name}" + f" in column {column.name}, out of a random sample of {sample_count} rows" + f" of table {column.table.name}" ) self._annotation = "sampled and suppressed" @@ -185,7 +163,7 @@ def get_estimated_counts(self, counts: list[int]) -> list[int]: def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" return { - "a": f'SRC_STATS["auto__{self.table_name}__{self.column_name}"]["results"]', + "a": f'SRC_STATS["auto__{self.column.table.name}__{self.column.name}"]["results"]', } def name(self) -> str: @@ -206,7 +184,7 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: qs = super().custom_queries() return { **qs, - f"auto__{self.table_name}__{self.column_name}": { + f"auto__{self.column.table.name}__{self.column.name}": { "query": self._query, "comments": [self._comment], }, @@ -350,23 +328,19 @@ def get_proposers( # pylint: disable=too-many-locals if len(columns) != 1: return [] column = columns[0] - column_name = column.name - table_name = column.table.name - table_sql = schema_qualified_name(table_name, engine) src_table = column.table dialect = engine.dialect - col = literal_column(f'"{column_name}"') src_table = column.table # preserves schema for schema-qualified databases generators = [] with engine.connect() as connection: stmt_count = ( select( - col.label("v"), - func.count(col).label("f"), # pylint: disable=E1102 + column.label("v"), + func.count(column).label("f"), # pylint: disable=E1102 ) .select_from(src_table) - .group_by(col) - .order_by(desc(func.count(col))) # pylint: disable=E1102 + .group_by(column) + .order_by(desc(func.count(column))) # pylint: disable=E1102 .limit(MAXIMUM_CHOICES + 1) ) results = connection.execute(stmt_count) @@ -375,62 +349,50 @@ def get_proposers( # pylint: disable=too-many-locals if vg.counts: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name, - column_name, + column, vg.cvs, vg.counts, dialect=dialect, - table_sql=table_sql, ), ] if vg.counts_not_suppressed: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name=table_name, - column_name=column_name, + column=column, values=vg.cvs_not_suppressed, counts=vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), ] inner = ( - select(col.label("v")) + select(column.label("v")) .select_from(src_table) .order_by(Random()) .limit(self.SAMPLE_COUNT) @@ -450,64 +412,52 @@ def get_proposers( # pylint: disable=too-many-locals if vg.counts: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name, - column_name, + column, vg.cvs, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), ] if vg.counts_not_suppressed: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name=table_name, - column_name=column_name, + column=column, values=vg.cvs_not_suppressed, counts=vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), ] return generators diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index e6bb11fc..c6142cba 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -3,23 +3,28 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlalchemy import Column, Engine, ForeignKey, MetaData, func, select +from sqlalchemy import Column, Engine, MetaData, func, select from sqlalchemy.types import Date, DateTime -from datafaker.db_utils import get_fk_column_between from datafaker.dialects import Random, SecondsDifference, StdDev -from datafaker.proposers.base import Buckets, Proposer, ProposerFactory, get_column_type +from datafaker.proposers.base import ( + Buckets, + ForeignKeyRelationship, + Proposer, + ProposerFactory, + RelatedColumn, + get_column_type, + make_foreign_key_relationship, +) from datafaker.providers import AnchoredProvider from datafaker.utils import get_property, procrustes -RelatedColumn = tuple[ForeignKey | None, Column] - def _set_roles_for_column( out: dict[str, list[RelatedColumn]], - fk: ForeignKey | None, + fk: ForeignKeyRelationship | None, column: Column, - column_config: Mapping, + columns_config: Mapping, ) -> None: """ Set new entries in ``out`` based on the roles ``column`` has. @@ -29,22 +34,22 @@ def _set_roles_for_column( that role and ``fk`` is the foreign key in the table ``column`` appears in that points to the table ``fcol`` appears in, or None if ``column`` and ``fcol`` are in the same table. - :param fk: Foreign key to the table ``column`` appears in (appears in - new entries set in ``out``). + :param fk: Foreign key relationship to the table ``column`` appears in + (appears in new entries set in ``out``). :param column: The column to be checked for roles. - :param column_config: The ``tables: : columns:`` section + :param columns_config: The ``tables: : columns:`` section of the ``config.yaml`` file. """ - roles: list[Any] = get_property(column_config, [column.name, "roles"], []) + roles: list[Any] = get_property(columns_config, [column.name, "roles"], []) for role in roles: - pair: RelatedColumn = (fk, column) + rc = RelatedColumn(column, fk) if role not in out: - out[role] = [pair] + out[role] = [rc] else: - out[role].append(pair) + out[role].append(rc) -def _get_roles( +def _get_all_relative_roles( config: Mapping, column: Column, ) -> dict[str, list[RelatedColumn]]: @@ -60,24 +65,95 @@ def _get_roles( """ table = column.table tables_config: dict[str, Any] = get_property(config, "tables", {}) - table_conf: dict[str, Any] = get_property( + columns_config: dict[str, Any] = get_property( tables_config, [str(table.name), "columns"], {} ) role_to_fk_columns: dict[str, list[RelatedColumn]] = {} for col in table.columns: - _set_roles_for_column(role_to_fk_columns, None, col, table_conf) + _set_roles_for_column(role_to_fk_columns, None, col, columns_config) # look for roles in related tables - if col.foreign_keys: - fk = list(col.foreign_keys)[0] - target_table = fk.column.table + if (fk_relationship := make_foreign_key_relationship(col)) is not None: ft_conf: dict[str, Any] = get_property( - tables_config, [str(target_table.name), "columns"], {} + tables_config, [fk_relationship.target_table_name(), "columns"], {} ) - for fcol in target_table.columns: - _set_roles_for_column(role_to_fk_columns, fk, fcol, ft_conf) + for fcol in fk_relationship.target_table().columns: + _set_roles_for_column( + role_to_fk_columns, fk_relationship, fcol, ft_conf + ) return role_to_fk_columns +def _not_anchored_on_ourselves(column: Column, fk_col: RelatedColumn) -> bool: + """ + Return whether these columns are the exact same thing. + + :param column: The column we are wanting to generate for. + :param fk_col: The column we are considering using as an anchor. + :return: False if ``fk_col`` targets the same column as ``column`` + not through a foreign key, so True if ``fk_col`` is through a FK + or if it doesn't target the same column. + """ + return not (fk_col.relationship is None and fk_col.target == column) + + +def _not_anchoring_nonnullable_to_nullable( + column: Column, fk_col: RelatedColumn +) -> bool: + """ + Return True if ``column`` is nullable or ``fk_col`` is not. + + We are counting ``fk_col`` as nullabe if either the target is nullable + or the foreign key itself. + + This is required if ``fk_col`` can be copied to ``col``. + """ + fk_nullable = ( + fk_col.relationship is not None + and fk_col.relationship.fk_column.nullable is not False + ) + fk_col_nullable = fk_nullable or fk_col.target.nullable + return column.nullable or not fk_col_nullable + + +def _not_copying_incompatible_fk(column: Column, fk_col: RelatedColumn) -> bool: + """Return True if ``column`` and ``fk_col`` are incompatible foreign key types.""" + target = fk_col.target + if not column.foreign_keys: + return not target.foreign_keys + if not target.foreign_keys: + return False + tfks = {c.target_fullname for c in target.foreign_keys} + cfks = {c.target_fullname for c in column.foreign_keys} + return tfks == cfks + + +def _get_roles( + config: Mapping, + column: Column, + role_name: str, +) -> list[RelatedColumn]: + """ + Get all the ``role_name`` roles in this table or a related table. + + :param role_name: The role to return. + :return: The list of all columns in the same table as ``column`` or in + tables directly related to that table that have the named role. This + list has the following columns filtered out: Nullable columns (if + ``column`` is nonnullable), foreign keys if ``column`` is not a + foreign key, non-foreign keys if ``column`` is a foreign key + or foreign keys if ``column`` is a foreign key to a different table. + """ + roles = _get_all_relative_roles(config, column) + if role_name not in roles: + return [] + return [ + fk_col + for fk_col in roles[role_name] + if _not_anchored_on_ourselves(column, fk_col) + and _not_anchoring_nonnullable_to_nullable(column, fk_col) + ] + + def coerce_to_datetime( d: datetime.date | datetime.datetime, tzdonor: datetime.datetime, @@ -103,7 +179,7 @@ def __init__( sd: float, mean: float, column: Column, - anchor: Column, + anchor_related: RelatedColumn, engine: Engine, buckets: Buckets | None = None, ): @@ -116,13 +192,13 @@ def __init__( :param column: The column to generate. :param anchor: The anchor column, which must be in the same table as column or a related table. - :param dialect: The dialect of the source database. + :param engine: The source database. :param buckets: The buckets for the intervals in the source database. """ super().__init__() self._sd = sd self._mean = mean - self._anchor = anchor + self._anchor_related = anchor_related self._column = column self._engine = engine self._provider = AnchoredProvider(metadata=metadata) @@ -134,27 +210,44 @@ def __init__( [(b - coerce_to_datetime(a, b)).total_seconds() for (a, b) in intervals] ) + @classmethod + def get_query( + cls, + column: Column, + anchor_related: RelatedColumn, + ) -> Any: + """Create the query for the summary of interval lengths.""" + anchor = anchor_related.target + return anchor_related.add_from_to_query( + select( + func.avg(SecondsDifference(column, anchor)).label("mean"), + StdDev(SecondsDifference(column, anchor)).label("sd"), + ) + ) + def function_name(self) -> str: """Get the name of the generator function to call.""" - if self._column.table == self._anchor.table: + if self._anchor_related.relationship is None: return "generic.anchored_provider.normal_date" return "generic.anchored_provider.normal_date_fk" def name(self) -> str: """Get the name of the generator.""" fname = self.function_name() - aname = self._anchor.name - atable = self._anchor.table - if atable == self._column.table: + aname = self._anchor_related.target.name + rel = self._anchor_related.relationship + if rel is None: return f"{fname} [anchored to {aname}]" - return f"{fname} [anchored to {aname} of table {atable.name}]" + atable = rel.target_table_name() + fk_name = rel.fk_column.name + return f"{fname} [anchored to {aname} of table {atable} (via {fk_name})]" def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" column = self._column - anchor = self._anchor - (fk_col, fk) = get_fk_column_between(column.table, anchor.table) - if fk_col is None or fk is None: + anchor = self._anchor_related.target + rel = self._anchor_related.relationship + if rel is None: return { "mean_seconds": ( f'SRC_STATS["auto__{column.table.name}"]' @@ -167,33 +260,34 @@ def nominal_kwargs(self) -> dict[str, Any]: "anchor": f'GENERATED_ROW["{anchor.name}"]', } key = f"auto__interval__{column.table.name}__{column.name}" + fk_col = rel.fk_column.name + on_col = rel.target_column.name return { "dst_db_conn": "dst_db_conn", "anchor_column": f'"{anchor.name}"', - "table": f'"{anchor.table.name}"', + "table": f'"{rel.target_table_name()}"', "mean_seconds": (f'SRC_STATS["{key}"]["results"][0]["mean"]'), "sd_seconds": (f'SRC_STATS["{key}"]["results"][0]["sd"]'), - "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', - "on_column": f'"{fk.column.name}"', + "anchor_row": f'GENERATED_ROW["{fk_col}"]', + "on_column": f'"{on_col}"', } def actual_kwargs(self) -> dict[str, Any]: """Get the kwargs (summary statistics) this generator was instantiated with.""" - column = self._column - anchor = self._anchor - (_, fk) = get_fk_column_between(column.table, anchor.table) - if fk is None: + if self._anchor_related.relationship is None: return { "mean_seconds": self._sd, "sd_seconds": self._mean, "anchor": "1970-01-01", } + anchor = self._anchor_related.target + on_col = self._anchor_related.relationship.target_column.name return { "anchor_column": anchor.name, "table": anchor.table.name, "mean_seconds": self._sd, "sd_seconds": self._mean, - "on_column": fk.column.name, + "on_column": on_col, } def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: @@ -202,10 +296,10 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: This is for anchors in the same table. """ - column = self._column - anchor = self._anchor - if column.table != anchor.table: + if self._anchor_related.relationship is not None: return {} + column = self._column + anchor = self._anchor_related.target mean_q = func.avg(SecondsDifference(column, anchor)) sd_q = StdDev(SecondsDifference(column, anchor)) @@ -243,23 +337,17 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: This is for anchors in a related table. """ column = self._column - anchor = self._anchor - if column.table == anchor.table: + rel = self._anchor_related + if rel.relationship is None: return {} - query = ( - select( - func.avg(SecondsDifference(column, anchor)).label("mean"), - StdDev(SecondsDifference(column, anchor)).label("sd"), - ) - .select_from(column.table) - .join(anchor.table) - ) + query = DateAfterProposer.get_query(column, rel) return { f"auto__interval__{column.table.name}__{column.name}": { "comments": [ "Mean and standard deviation of the length of time between" - f" column {anchor.name} of table {anchor.table.name}" - f" and column {column.name} of table {column.table.name}." + f" column {rel.target.name} of table {rel.target.table.name}" + f" and column {column.name} of table {column.table.name}" + f" (joined through {rel.relationship.fk_column.name})" ], "query": str( query.compile( @@ -288,7 +376,8 @@ def generate_intervals( :param count: The number of intervals to generate. :return: Each pair consists of an anchor and the generated datetime. """ - query = select(self._anchor).select_from(self._anchor.table) + anchor = self._anchor_related.target + query = select(anchor).select_from(anchor.table) query = query.order_by(Random()).limit(count) with self._engine.connect() as conn: anchors = conn.execute(query).scalars().all() @@ -312,18 +401,14 @@ def __init__(self, config: Mapping, metadata: MetaData): self._config = config self._metadata = metadata - def make_date_after_proposers( - self, engine: Engine, column: Column, anchor: Column + def _make_date_after_proposers( + self, + engine: Engine, + column: Column, + anchor_related: RelatedColumn, ) -> list[DateAfterProposer]: """Create a ``DateAfterProposer`` object.""" - query = select( - func.avg(SecondsDifference(column, anchor)).label("mean"), - StdDev(SecondsDifference(column, anchor)).label("sd"), - ).select_from(column.table) - join_tables = [] - if anchor.table != column.table: - join_tables = [anchor.table] - query = query.join(anchor.table) + query = DateAfterProposer.get_query(column, anchor_related) with engine.connect() as connection: result = connection.execute(query).first() if result is None or result.sd is None: @@ -331,8 +416,8 @@ def make_date_after_proposers( buckets = Buckets.make_buckets( engine, column.table, - SecondsDifference(column, anchor), - join_tables, + SecondsDifference(column, anchor_related.target), + anchor_related, ) return [ DateAfterProposer( @@ -340,7 +425,7 @@ def make_date_after_proposers( result.sd, result.mean, column, - anchor, + anchor_related, engine=engine, buckets=buckets, ) @@ -358,14 +443,151 @@ def get_proposers( ct = get_column_type(column) if not isinstance(ct, (Date, DateTime)): return [] - roles = _get_roles(self._config, column) - if "start" not in roles: + other_start_columns = _get_roles(self._config, column, "start") + return [ + prop + for anchor in other_start_columns + for prop in self._make_date_after_proposers(engine, column, anchor) + ] + + +class CopyProposer(Proposer): + """Proposer that proposes copying another column.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + metadata: MetaData, + column: Column, + anchor_related: RelatedColumn, + engine: Engine, + ): + """ + Initialise a date after proposer. + + :param metadata: The metadata of the source database. + :param column: The column to generate. + :param anchor: The anchor column, which must be in the same table as column + or a related table. + :param engine: The source database. + :param buckets: The buckets for the intervals in the source database. + """ + super().__init__() + self._anchor_related = anchor_related + self._column = column + self._engine = engine + self._provider = AnchoredProvider(metadata=metadata) + + def function_name(self) -> str: + """Get the name of the generator function to call.""" + if self._anchor_related.relationship is None: + return "generic.anchored_provider.copy" + return "generic.anchored_provider.copy_fk" + + def name(self) -> str: + """Get the name of the generator.""" + fname = self.function_name() + aname = self._anchor_related.target.name + rel = self._anchor_related.relationship + if rel is None: + return f"{fname} [from {aname}]" + atable = rel.target_table_name() + on_col = rel.fk_column.name + return f"{fname} [from {aname} of table {atable} (via {on_col})]" + + def nominal_kwargs(self) -> dict[str, Any]: + """Get the arguments to be entered into ``config.yaml``.""" + anchor = self._anchor_related.target + rel = self._anchor_related.relationship + if rel is None: + return { + "anchor": f'GENERATED_ROW["{anchor.name}"]', + } + fk_col = rel.fk_column + on_col = rel.target_column + return { + "dst_db_conn": "dst_db_conn", + "anchor_column": f'"{anchor.name}"', + "table": f'"{rel.target_table_name()}"', + "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', + "on_column": f'"{on_col.name}"', + } + + def actual_kwargs(self) -> dict[str, Any]: + """Get the kwargs (summary statistics) this generator was instantiated with.""" + anchor = self._anchor_related.target + if self._anchor_related.relationship is None: + return { + "anchor": "1970-01-01", + } + on_col = self._anchor_related.relationship.target_column + return { + "anchor_column": anchor.name, + "table": anchor.table.name, + "on_column": on_col.name, + } + + def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: + """No source data required.""" + return {} + + def custom_queries(self) -> dict[str, dict[str, Any]]: + """No source data required.""" + return {} + + def fit(self, default: float = -1) -> float: + """Get this generator's fit against the real data.""" + return default + + def generate_data(self, count: int) -> list[Any]: + """Generate ``count`` random data points for this column.""" + anchor = self._anchor_related.target + query = select(anchor).select_from(anchor.table) + query = query.order_by(Random()).limit(count) + with self._engine.connect() as conn: + return list(conn.execute(query).scalars().all()) + + +class CopyProposerFactory(ProposerFactory): + """Proposer that proposes copying another column.""" + + def __init__(self, config: Mapping, metadata: MetaData): + """Initialize ``CopyProposerFactory``.""" + super().__init__() + self._config = config + self._metadata = metadata + + def make_copy_fk_proposers( + self, engine: Engine, column: Column, anchor: RelatedColumn + ) -> list[CopyProposer]: + """Create a ``CopyProposer`` object.""" + return [ + CopyProposer( + self._metadata, + column, + anchor, + engine=engine, + ) + ] + + def _can_copy_type(self, copy_from, copy_to) -> bool: + return type(copy_from) is type(copy_to) + + def get_proposers( + self, + columns: list[Column], + engine: Engine, + ) -> Sequence[Proposer]: + """Get all proposers of dates that might be anchored to another column.""" + if len(columns) != 1: return [] - other_start_columns = [ - fk_col for fk_col in roles["start"] if fk_col[1] != column + column = columns[0] + fk_cols = _get_roles(self._config, column, "source") + other_source_columns = [ + fk_col for fk_col in fk_cols if _not_copying_incompatible_fk(column, fk_col) ] return [ prop - for anchor in other_start_columns - for prop in self.make_date_after_proposers(engine, column, anchor[1]) + for anchor in other_source_columns + for prop in self.make_copy_fk_proposers(engine, column, anchor) ] diff --git a/datafaker/proposers/mimesis.py b/datafaker/proposers/mimesis.py index 451ad792..dbb9282f 100644 --- a/datafaker/proposers/mimesis.py +++ b/datafaker/proposers/mimesis.py @@ -5,7 +5,7 @@ import mimesis import mimesis.locales -from sqlalchemy import Column, Engine, cast, extract, func, literal_column, select +from sqlalchemy import Column, Engine, cast, extract, func, select from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.types import Date, DateTime, Integer, Numeric, String, Time @@ -187,8 +187,7 @@ def make_singleton( cls, column: Column, engine: Engine, function_name: str ) -> Sequence[Proposer]: """Make the appropriate generation configuration for this column.""" - col_expr = literal_column(column.name) - year_expr = cast(extract("year", col_expr), Integer()) + year_expr = cast(extract("year", column), Integer()) min_expr = func.min(year_expr) # pylint: disable=E1111 max_expr = func.max(year_expr) # pylint: disable=E1111 stmt = select(min_expr.label("start"), max_expr.label("end")).select_from( diff --git a/datafaker/proposers/partitioned.py b/datafaker/proposers/partitioned.py index 595fcf81..6f8bc669 100644 --- a/datafaker/proposers/partitioned.py +++ b/datafaker/proposers/partitioned.py @@ -28,7 +28,7 @@ CovariateQuery, MultivariateNormalProposerFactory, ) -from datafaker.utils import T, get_property, logger +from datafaker.utils import T, get_property, logger, text_list NumericType = Union[int, float] @@ -37,23 +37,6 @@ MAXIMUM_CHOICES = 500 -def text_list(items: Iterable[str]) -> str: - """Concatenate the items with commas and one "and".""" - item_i = iter(items) - try: - last_item = next(item_i) - except StopIteration: - return "" - try: - so_far = next(item_i) - except StopIteration: - return last_item - for item in item_i: - so_far += ", " + last_item - last_item = item - return so_far + " and " + last_item - - @dataclass class RowPartition: """A partition where all the rows have the same pattern of NULLs.""" diff --git a/datafaker/providers.py b/datafaker/providers.py index 0263ef56..55faabf6 100644 --- a/datafaker/providers.py +++ b/datafaker/providers.py @@ -693,8 +693,8 @@ def normal_date_fk( :param sd_seconds: Standard deviation of the intervals' lengths in seconds. :param table: The name of the table for the anchor column. :param on_column: The name of the column in the foreign table - that ``row`` must match. - :param row: The value in the ``on_column`` column in table ``table`` + that ``anchor_row`` must match. + :param anchor_row: The value in the ``on_column`` column in table ``table`` for the row providing the start of the interval. :param anchor_column: The name of the column in ``table`` providing the start of the interval. @@ -711,3 +711,34 @@ def normal_date_fk( if not isinstance(out, (str, dt.date, dt.datetime)): return None return self.normal_date(mean_seconds, sd_seconds, out) + + def copy(self, anchor: Any) -> Any: + """Copy another column from the same table.""" + return anchor + + def copy_fk( + self, + dst_db_conn: Connection, + table: str, + on_column: str, + anchor_row: Any, + anchor_column: str, + ) -> Any: + """ + Copy a column from a different table. + + :param dst_db_conn: Connection to the destination database. + :param table: The name of the table for the anchor column. + :param on_column: The name of the column in the foreign table + that ``anchor_row`` must match. + :param anchor_row: The value in the ``on_column`` column in table ``table`` + for the row providing the source to copy. + :param anchor_column: The name of the column in ``table`` to copy. + :return: The value to be copied. + """ + mt = self._metadata.tables[table] + query = select(mt.c[anchor_column].label("out")).where( + mt.c[on_column] == anchor_row, + ) + anchor = dst_db_conn.execute(query).first() + return getattr(anchor, "out", None) diff --git a/datafaker/remove.py b/datafaker/remove.py index 0949e048..a6eae71c 100644 --- a/datafaker/remove.py +++ b/datafaker/remove.py @@ -68,5 +68,9 @@ def remove_db_tables(metadata: Optional[MetaData]) -> None: ) ) if metadata is None: - metadata = get_metadata(dst_engine, schema_name=schema_name) + metadata = get_metadata( + dst_engine, + schema_name=schema_name, + views_to_tables=False, + ) metadata.drop_all(dst_engine) diff --git a/datafaker/serialize_metadata.py b/datafaker/serialize_metadata.py index 35a066b7..fc2b555c 100644 --- a/datafaker/serialize_metadata.py +++ b/datafaker/serialize_metadata.py @@ -6,6 +6,7 @@ from typing import Any import parsy +import snowflake.sqlalchemy.custom_types as snowflake_types from sqlalchemy import Column, Dialect, Engine, ForeignKey, MetaData, Table from sqlalchemy.dialects import mssql, oracle, postgresql from sqlalchemy.sql import schema, sqltypes @@ -103,7 +104,7 @@ def time_type(type_: type, tz_type: type) -> ParserType: Make a parser for a SQL date/time type. Parses TYPE_NAME, TYPE_NAME(32), TYPE_NAME WITH TIME ZONE - or TYPE_NAME(32) WITH TIME ZONE + or TYPE_NAME(32) WITH TIME ZONE, also WITHOUT :param type_: The SQLAlchemy type we would like to parse. :param tz_type: The type to instantiate when precision or timezone is @@ -204,6 +205,15 @@ def _mssql_varbinary_parser() -> typing.Generator[ParserType, None, typing.Any]: string_type(sqltypes.VARCHAR), string_type(sqltypes.NVARCHAR), string_type(sqltypes.TEXT), + parsy.string("TIMESTAMP_NTZ").result(sqltypes.DATETIME), + parsy.string("TIMESTAMPNTZ").result(sqltypes.DATETIME), + parsy.string("TIMESTAMP_LTZ").result(snowflake_types.TIMESTAMP_LTZ), + parsy.string("TIMESTAMPLTZ").result(snowflake_types.TIMESTAMP_LTZ), + parsy.string("TIMESTAMP WITH LOCAL TIME ZONE").result( + snowflake_types.TIMESTAMP_LTZ + ), + parsy.string("TIMESTAMP_TZ").result(lambda: sqltypes.DATETIME(timezone=True)), + parsy.string("TIMESTAMPTZ").result(lambda: sqltypes.DATETIME(timezone=True)), time_type(sqltypes.TIMESTAMP, postgresql.types.TIMESTAMP), time_type(sqltypes.TIME, postgresql.types.TIME), ) diff --git a/datafaker/utils.py b/datafaker/utils.py index a6a15321..e7ce255e 100644 --- a/datafaker/utils.py +++ b/datafaker/utils.py @@ -457,6 +457,23 @@ def underline_error(e: SyntaxError) -> str: return "\n" + " " * start + "^" * (end - start) +def text_list(items: Iterable[str], empty="") -> str: + """Concatenate the items with commas and one "and".""" + item_i = iter(items) + try: + last_item = next(item_i) + except StopIteration: + return empty + try: + so_far = next(item_i) + except StopIteration: + return last_item + for item in item_i: + so_far += ", " + last_item + last_item = item + return so_far + " and " + last_item + + def gather_from_ast( errors: MutableSequence[tuple], name: str, diff --git a/docs/source/docker.rst b/docs/source/docker.rst index 62fcfe0a..9c21bb2b 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -23,7 +23,7 @@ and stores the files produced in a directory called ``output``: .. code-block:: shell mkdir output - docker run --rm --user $(id -u):$(id -g) --network host -e SRC_SCHEMA=public -e SRC_DSN=postgresql://tim:tim@localhost:5432/pagila -itv ./output:data datafaker + docker run --rm --user $(id -u):$(id -g) --network host -e SRC_SCHEMA=public -e SRC_DSN=postgresql://tim:tim@localhost:5432/pagila -itv ./output:/data datafaker You do need to create the output folder first. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index a5700f02..82bc9dbb 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -1,19 +1,52 @@ FAQ === -Can datafaker work with two different schemas? +Can DataFaker work with two different schemas? ********************************************** -datafaker can only work with a single source schema and a single destination schema at a time. -However, you can choose for the destination schema to have a different name to the source schema by setting the ``DST_SCHEMA`` environment variable. +DataFaker can only work with a single source schema or a single destination schema at a time. +It is possible to use the same database for source and destination by choosing a different schema for source and destination, +for example by setting ``DST_SCHEMA=fake``. Which DBMSs does datafaker support? *********************************** -* datafaker most fully supports **PostgresSQL**, which it uses for its end-to-end functional tests. -* datafaker also supports **MariaDB**, as long as you don't set ``use-asyncio: true`` in your config. -* datafaker *might* work with **SQLite** but this is largely untested. -* datafaker may also work with SQL Server. - To connect to SQL Server, you will need to install `pyodbc `_ and an `ODBC driver `_, after which you should be able to use a DSN setting similar to ``SRC_DSN="mssql+pyodbc://username:password@hostname/dbname?driver=ODBC Driver 18 for SQL Server"``. +PostgreSQL +^^^^^^^^^^ -Please open a GitHub issue if you would like to see support for another DBMS. +Fully supported. + +Microsoft SQL Server +^^^^^^^^^^^^^^^^^^^^ + +Supported. Microsoft Transact SQL probably works as well. +You need to install an `ODBC driver `_. + +The DSN format is similar to: + +``mssql+pyodbc://:@127.0.0.1:1433/?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes`` + +DuckDB +^^^^^^ + +Supported. +If you want to produce Parquet or CSV files as your output format, the easiest way is via a DuckDB database. +You do not need to install DuckDB to do this, simply use a destination DSN like ``duckdb:///./fake.db``, +then use ``datafaker dump-data --output my-csv-folder`` or ``datafaker dump-data --parquet --output my-parquet-folder``. + +An in-memory DuckDB database can be used as an empty database that you can augment with parquet files: + +``datafaker make-tables --parquet-dir my-parquet-input-folder`` + +Snowflake +^^^^^^^^^ + +Currently experimental support. If the input database includes the type ``TIMEZONE_LTZ`` (or equivalent) +then the output database must also be a Snowflake database. + +Other +^^^^^ + +In principle DataFaker can support any database that SqlAlchemy supports. +However, each new database supported so far has required significant effort, +so no guarantees can be made for, for example, MySQL or MariaDB. diff --git a/docs/source/quickstart.rst b/docs/source/quickstart.rst index def95218..7c696958 100644 --- a/docs/source/quickstart.rst +++ b/docs/source/quickstart.rst @@ -78,6 +78,30 @@ The first job is to read the structure of the source database: This will create a file called ``orm.yaml``. You should not need to edit this file. +Troubleshooting: What if my tables do not appear in the orm.yaml file? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If the tables you expected to see do not all appear in the ``orm.yaml`` file, +perhaps these are not actual tables, but views. + +DataFaker can copy views as though they are tables. If you want this behaviour +(for example, if you have no access to the underlying tables) you can use +the ``--views-to-tables`` option: + +.. code-block:: console + + $ datafaker make-tables --views-to-tables + +This will produce definitions of each view as well as each table. +This is not a perfect system; DataFaker will guess which columns represent +primary or foreign keys as this information is not present in views. +So, after running ``make-tables`` with the ``--view-to-tables`` option, +please check the ``orm.yaml`` file produced: + +* make sure that each table has its correct primary key (DataFaker will have warned you if DataFaker found no or multiple primary keys for any table) +* make sure that all foreign key columns are correctly marked +* delete any tables you don't want and have no foreign keys pointing to them (or you could mark them as ignored in the following ``configure-tables`` stage) + Configuring table types ----------------------- diff --git a/examples/omop-mssql/README.md b/examples/omop-mssql/README.md index 7c0bbfed..64662a35 100644 --- a/examples/omop-mssql/README.md +++ b/examples/omop-mssql/README.md @@ -51,8 +51,6 @@ PostgreSQL uses `SERIAL` for autoincrement columns. The code already strips `SER [datafaker/utils.py:297-309](../../datafaker/utils.py) toggles `connection.autocommit` directly on the DBAPI connection before executing `SET` commands. The availability and behaviour of `autocommit` differs between `psycopg2`, `pyodbc`, and `pymssql`, so this would need testing or an abstraction per driver. -**Deferred.** `set_db_settings` is only ever called for DuckDB connections (`parquet_dir` source feature); MS-SQL connections never reach it. Fixing the `autocommit` toggle in isolation would also leave the `SET {k} TO {v}` SQL syntax broken for MS-SQL (which requires `SET {k} {v}`). Both issues should be addressed together if `set_db_settings` is ever extended to MS-SQL. - ## Setup ### 1. Install the MS-SQL Python extras diff --git a/poetry.lock b/poetry.lock index a34332f0..81be50d4 100644 --- a/poetry.lock +++ b/poetry.lock @@ -251,6 +251,44 @@ d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "boto3" +version = "1.43.106" +description = "The AWS SDK for Python (Boto3)" +optional = false +python-versions = ">=3.10" +files = [ + {file = "boto3-1.43.106-py3-none-any.whl", hash = "sha256:fdf2e304f9e8864b8613591b18b36ee995560c9666033622eda1cb9057866880"}, + {file = "boto3-1.43.106.tar.gz", hash = "sha256:c11ad4c429a983493ba10014c7af9831a455c2c0eea91c1cefff74530e480277"}, +] + +[package.dependencies] +botocore = ">=1.43.106,<1.44.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.19.0,<0.20.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.43.106" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.10" +files = [ + {file = "botocore-1.43.106-py3-none-any.whl", hash = "sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78"}, + {file = "botocore-1.43.106.tar.gz", hash = "sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3" + +[package.extras] +crt = ["awscrt (==0.36.0)"] + [[package]] name = "certifi" version = "2026.7.22" @@ -262,6 +300,118 @@ files = [ {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] +[[package]] +name = "cffi" +version = "2.1.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.10" +files = [ + {file = "cffi-2.1.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be"}, + {file = "cffi-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9"}, + {file = "cffi-2.1.1-cp310-cp310-win32.whl", hash = "sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41"}, + {file = "cffi-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa"}, + {file = "cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3"}, + {file = "cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0"}, + {file = "cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"}, + {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"}, + {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"}, + {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7"}, + {file = "cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac"}, + {file = "cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d"}, + {file = "cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13"}, + {file = "cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c"}, + {file = "cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48"}, + {file = "cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f"}, + {file = "cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4"}, + {file = "cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e"}, + {file = "cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7"}, + {file = "cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac"}, + {file = "cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960"}, + {file = "cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5"}, + {file = "cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66"}, + {file = "cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3"}, + {file = "cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692"}, + {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"}, +] + +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + [[package]] name = "cfgv" version = "3.5.0" @@ -545,6 +695,81 @@ files = [ [package.extras] dev = ["black (==22.3.0)", "hypothesis (==6.60.0)", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"] +[[package]] +name = "cryptography" +version = "50.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +files = [ + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"}, + {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11\""} + +[package.extras] +ssh = ["bcrypt (>=3.1.5)"] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -997,6 +1222,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.1.0" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.9" +files = [ + {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, + {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, +] + [[package]] name = "json-schema-for-humans" version = "1.5.1" @@ -1756,6 +1992,22 @@ wcwidth = ">=0.3.5" [package.extras] tests = ["pytest (>=9)", "pytest-cov", "pytest-lazy-fixtures"] +[[package]] +name = "psycopg2" +version = "2.9.13" +description = "psycopg2 - Python-PostgreSQL Database Adapter" +optional = false +python-versions = ">=3.10" +files = [ + {file = "psycopg2-2.9.13-cp310-cp310-win_amd64.whl", hash = "sha256:7d48416f6a4823ada9b33771085331b842b553df88435701bff5ddb4469905de"}, + {file = "psycopg2-2.9.13-cp311-cp311-win_amd64.whl", hash = "sha256:d16e7a5f5e400ac51ca953d42255804eff6c8a9650b1a2074f6ca6261d740382"}, + {file = "psycopg2-2.9.13-cp312-cp312-win_amd64.whl", hash = "sha256:10f7408b34412e8c0d4f8b1565541f1d651b1d00447857e5d8561b38f5c1a738"}, + {file = "psycopg2-2.9.13-cp313-cp313-win_amd64.whl", hash = "sha256:165e25c1b0e616a1f28080c5c68bd2dc015051d83c90240b2171d3e76ca2b5ff"}, + {file = "psycopg2-2.9.13-cp314-cp314-win_amd64.whl", hash = "sha256:a6f54fd8e0024f35240866b5dfff9ead2a0dbd33b8096eec438bc6093412842d"}, + {file = "psycopg2-2.9.13-cp315-cp315-win_amd64.whl", hash = "sha256:0d2fc7eedfaca0586dcf1476454598428d0d8471d3b5cb55f92015a5f9d0af40"}, + {file = "psycopg2-2.9.13.tar.gz", hash = "sha256:d36784fc2dae69523ba4b79c7d1d1b4d6e83e87836874f111262f4db940b16a6"}, +] + [[package]] name = "psycopg2-binary" version = "2.9.12" @@ -1832,6 +2084,17 @@ files = [ {file = "psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c"}, ] +[[package]] +name = "pycparser" +version = "3.0" +description = "C parser in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, +] + [[package]] name = "pydantic" version = "1.10.26" @@ -1917,6 +2180,23 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyjwt" +version = "2.15.1" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"}, + {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"}, +] + +[package.dependencies] +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] + [[package]] name = "pylint" version = "3.3.9" @@ -2034,6 +2314,25 @@ files = [ {file = "pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05"}, ] +[[package]] +name = "pyopenssl" +version = "26.4.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c"}, + {file = "pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7"}, +] + +[package.dependencies] +cryptography = ">=49.0.0,<51" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\""} + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] +test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] + [[package]] name = "pytest" version = "9.1.1" @@ -2408,6 +2707,23 @@ sphinx = ["sphinx (>=4.0,<6.0)"] testing = ["coverage-conditional-plugin (>=0.5)", "coverage[toml] (>=6.0)", "pytest (>=6.0)", "pytest-cov (>=3.0)", "pytest-mock (>=3.7)", "pytest-randomly (>=3.0)", "pytest-sugar (>=0.9.5)"] toml = ["tomli (>=2.0,<3.0)"] +[[package]] +name = "s3transfer" +version = "0.19.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.10" +files = [ + {file = "s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25"}, + {file = "s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + [[package]] name = "scramp" version = "1.4.12" @@ -2474,6 +2790,102 @@ files = [ {file = "snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260"}, ] +[[package]] +name = "snowflake-connector-python" +version = "4.8.0" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "snowflake_connector_python-4.8.0-cp310-cp310-macosx_15_0_arm64.whl", hash = "sha256:5ac7214337e2ca4797a63b12e2d6778b85b95bacbb1e8892e8ef5cdb2696bcdb"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-macosx_15_0_x86_64.whl", hash = "sha256:e3de20df18a0279cc913ecd073b79eae860fa00e3d0e8cb9cfb276ecf85fe319"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:a887838e59afa9d6d5f0fb52f55fcdb8726987acce646daa9aa3c698a3c99690"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b95cec73e136a3a373bf19bc19ecc5707557ae9df02674665240cfd46ea78820"}, + {file = "snowflake_connector_python-4.8.0-cp310-cp310-win_amd64.whl", hash = "sha256:3566b6884073dd1c36905c875aa3013991ae814a75f539c6ea1f2e8a1f72dd41"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-macosx_15_0_arm64.whl", hash = "sha256:3b38204c9c8a9c985cdf36df4eaa16f7d0b8a14f86410f39ab7b49794df4cf03"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-macosx_15_0_x86_64.whl", hash = "sha256:3fdea6d45bd22142293ece1224f2ad3a3c169a2f4a768d4be828faf09a640b2b"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bf2304eedfd94694291a71c96cddb4490e474d2c232625f528813f5caa79c098"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:bf8d819fd2c943d9be9d521fb6259b28dca05844e6a4d585a9f6e287141ae6dd"}, + {file = "snowflake_connector_python-4.8.0-cp311-cp311-win_amd64.whl", hash = "sha256:1ac5bc235c9f1124bfa1b14413f340ef2bdf7c28746af37280eae3ad7877afa8"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_arm64.whl", hash = "sha256:2fde1216c4c1e4015ad7e39b7b39b6a44a29551f4d7a3dfc64b0905e66dc24ac"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-macosx_15_0_x86_64.whl", hash = "sha256:c40a058f87dfbca1e65122932afb41c42cb9c3b7cb12826204527b41dd0f39ef"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f2a3cbcd3e6c4a449165f78bb331e1570ce6026cc21a2673313b4912d80745c0"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c80df2fc5e16f3a49e6d16b1cdca0358308c1db2eae22d82aa7c038c80cc913"}, + {file = "snowflake_connector_python-4.8.0-cp312-cp312-win_amd64.whl", hash = "sha256:b9bddddc713aaa20dc5ff5635896f66e50c0fed88b6a905b088e727d046e361d"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-macosx_15_0_arm64.whl", hash = "sha256:ebcbad6b16f8228f28bf62b443b6f111d11bceea2612efa2004eccb827923e6b"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-macosx_15_0_x86_64.whl", hash = "sha256:1cc888b81b709781eb855bbee062f06a9f52427ab8f9a39a31f39de475e39465"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:2a59396eade5b7b18cad0ac1fa2882a5ddfa5153e56e16c9d3d4a9d5e2d92332"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f978f897f85fd39cc8f768971c4667f8ca0278ea369373dfc459ae008c9472a5"}, + {file = "snowflake_connector_python-4.8.0-cp313-cp313-win_amd64.whl", hash = "sha256:131db582e383b57af0b345f212c9afba87637e8ef68494b9b4282e6713030dc9"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-macosx_15_0_arm64.whl", hash = "sha256:7a1566b0fcae33f2c7ec1bbd09965d8045a9388e94d333a88d03ddcfb6d4550a"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-macosx_15_0_x86_64.whl", hash = "sha256:8db3eea888247e4b41b55b8b419613eb1c4c76ec4984341543f849bf9532ae1b"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:39fe1794c63508fb25fda1a57cdf00293dc0630b32d4cda86d5bed2da09eb256"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:43583fb372a3bb698c2ce4717d2598e5452b40543a59b8095146a98f8ecd7408"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314-win_amd64.whl", hash = "sha256:6ba170f3584a425a1a5eff0e53ed656d45db9ef513d9232789cf58c5254b8194"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-macosx_15_0_arm64.whl", hash = "sha256:03e248f510ae82a3b190146c060e865905e5e07df6cf3e365ede66c67de136b2"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-macosx_15_0_x86_64.whl", hash = "sha256:7612ef5b4a93438fec6fff0e0d2541302f29001d3e77b861140d00d9085c0dce"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bccf23cd780c85bfa74982f286d2aacd663ba0135ba7441a13e98cabdb2def1e"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:da28999b3d6cfc4860c8c537cfb02968e9784f3ec8fe50d92a7348133ea75362"}, + {file = "snowflake_connector_python-4.8.0-cp314-cp314t-win_amd64.whl", hash = "sha256:6c0d7be79b333c1fa85d2860928f286a11abe5789d007dbc6d940313a744f8af"}, + {file = "snowflake_connector_python-4.8.0.tar.gz", hash = "sha256:1bf40aa8684eb0769fedeb53a702b274472571812835d5a03fb9dcf9254333b9"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +boto3 = ">=1.24" +botocore = ">=1.24" +certifi = ">=2024.7.4" +charset_normalizer = ">=2,<4" +cryptography = ">=46.0.5" +filelock = ">=3.5,<4" +idna = ">=3.7,<4" +packaging = "*" +platformdirs = ">=2.6.0,<5.0.0" +pyjwt = ">=2.10.1,<3.0.0" +pyOpenSSL = ">=25.3.0" +pytz = "*" +requests = ">=2.32.4,<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing_extensions = ">=4.3,<5" + +[package.extras] +azure = ["azure-identity (>=1.16)"] +boto = ["boto3 (>=1.24)", "botocore (>=1.24)"] +development = ["Cython (>=3.1)", "azure-identity (>=1.16)", "coverage", "mitmproxy (>=12.0.0)", "more-itertools", "numpy (<=2.4.3)", "pendulum (!=2.1.1)", "pexpect", "pytest (>=8.4,<9)", "pytest-asyncio (>=1.4.0,<2)", "pytest-cov", "pytest-rerunfailures (<16.0)", "pytest-timeout", "pytest-xdist", "pytzdata", "responses"] +pandas = ["pandas (>=1.0.0,<3.0.0)", "pandas (>=2.1.2,<3.0.0)", "pyarrow (>=14.0.1)", "pyarrow (>=14.0.1,<24)"] +secure-local-storage = ["keyring (>=23.1.0,<26.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.11.1" +description = "Snowflake SQLAlchemy Dialect" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake_sqlalchemy-1.11.1-py3-none-any.whl", hash = "sha256:b64ab947469bf25b3df3f88c5b92f17a938c9d47b5b09035551614f3ff1f44d9"}, + {file = "snowflake_sqlalchemy-1.11.1.tar.gz", hash = "sha256:942b203de0404b79550c4098ca851aee2b0b22f695426b073e58d71772e38d51"}, +] + +[package.dependencies] +snowflake-connector-python = "<5.0.0" +sqlalchemy = ">=1.4.19" + +[package.extras] +development = ["alembic", "numpy", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytz", "setuptools", "syrupy"] +pandas = ["snowflake-connector-python[pandas]"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + [[package]] name = "sphinx" version = "5.3.0" @@ -3174,4 +3586,4 @@ docs = ["sphinx-rtd-theme", "sphinxcontrib-napoleon"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.14" -content-hash = "32e39aeac70fde68398a8a5d467c29499cd74d177beda122a19f4641689f71c3" +content-hash = "5762718d243ff3e673d19efc74904d0302ac47e444598edd25132b10c2498673" diff --git a/pyproject.toml b/pyproject.toml index 09096059..1d44b2b7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datafaker" -version = "0.5.3" +version = "0.5.4" description = "Generates fake SQL data" authors = ["Tim Band <3266052+tim-band@users.noreply.github.com>"] license = "MIT" @@ -40,6 +40,8 @@ fastparquet = "^2024.11.0" duckdb-sqlalchemy = "^1.5.2.2" ty = "^0.0.43" colorama = "^0.4.6" +snowflake-sqlalchemy = "^1.11.1" +psycopg2 = "^2.9.13" [tool.poetry.group.dev.dependencies] isort = "^5.10.1" diff --git a/tests/examples/date.sql b/tests/examples/date.sql index a0155a9d..2f531718 100644 --- a/tests/examples/date.sql +++ b/tests/examples/date.sql @@ -8,31 +8,47 @@ CREATE TABLE public.person ( id INTEGER NOT NULL, name TEXT NOT NULL, timestamp_of_birth TIMESTAMP WITH TIME ZONE NOT NULL, - date_of_birth DATE NOT NULL + date_of_birth DATE NOT NULL, + parent_of INTEGER ); ALTER TABLE ONLY public.person ADD CONSTRAINT person_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.person + ADD CONSTRAINT parent_of_fkey FOREIGN KEY (parent_of) REFERENCES public.person(id); ALTER TABLE public.person OWNER TO postgres; -INSERT INTO public.person VALUES (1, 'Bobby', '1991-01-08 11:12:13+00:00', '1991-01-08'); -INSERT INTO public.person VALUES (2, 'Mary', '1989-03-04 20:19:18+00:00', '1989-03-04'); +INSERT INTO public.person VALUES (1, 'Bobby', '1991-01-08 11:12:13+00:00', '1991-01-08', NULL); +INSERT INTO public.person VALUES (2, 'Mary', '1989-03-04 20:19:18+00:00', '1989-03-04', NULL); +INSERT INTO public.person VALUES (3, 'Elizabeth', '1951-10-14 00:12:12+00:00', '1951-10-14', 2); CREATE TABLE public.happening ( id INTEGER NOT NULL, name TEXT NOT NULL, person_id INTEGER NOT NULL, at_time TIMESTAMP WITH TIME ZONE NOT NULL, - at_date DATE NOT NULL + at_date DATE NOT NULL, + other_person_id INTEGER, + comment TEXT, + previous_happening_id INTEGER ); ALTER TABLE ONLY public.happening ADD CONSTRAINT happening_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.happening ADD CONSTRAINT person_id_fkey FOREIGN KEY (person_id) REFERENCES public.person(id); +ALTER TABLE ONLY public.happening + ADD CONSTRAINT other_person_id_fkey FOREIGN KEY (other_person_id) REFERENCES public.person(id); +ALTER TABLE ONLY public.happening + ADD CONSTRAINT previous_happening_id_fkey FOREIGN KEY (previous_happening_id) REFERENCES public.happening(id); ALTER TABLE public.happening OWNER TO postgres; -INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20'); -INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20'); -INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23'); -INSERT INTO public.happening VALUES (4, 'steppen in gum', 2, '2003-11-03 04:15:26+00:00', '2003-11-03'); +INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20', NULL, NULL, NULL); +INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL, 1); +INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL, 1); +INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL, NULL); + +CREATE VIEW public.happening_view (id, name, person_id) + AS SELECT id, name, person_id FROM public.happening; + +ALTER VIEW public.happening_view OWNER TO postgres; diff --git a/tests/examples/duckdb/config.yaml b/tests/examples/duckdb/instrument/config.yaml similarity index 100% rename from tests/examples/duckdb/config.yaml rename to tests/examples/duckdb/instrument/config.yaml diff --git a/tests/examples/duckdb/manufacturer.parquet b/tests/examples/duckdb/instrument/manufacturer.parquet similarity index 100% rename from tests/examples/duckdb/manufacturer.parquet rename to tests/examples/duckdb/instrument/manufacturer.parquet diff --git a/tests/examples/duckdb/model.parquet b/tests/examples/duckdb/instrument/model.parquet similarity index 100% rename from tests/examples/duckdb/model.parquet rename to tests/examples/duckdb/instrument/model.parquet diff --git a/tests/examples/duckdb/orm.yaml b/tests/examples/duckdb/instrument/orm.yaml similarity index 100% rename from tests/examples/duckdb/orm.yaml rename to tests/examples/duckdb/instrument/orm.yaml diff --git a/tests/examples/duckdb/player.parquet b/tests/examples/duckdb/instrument/player.parquet similarity index 100% rename from tests/examples/duckdb/player.parquet rename to tests/examples/duckdb/instrument/player.parquet diff --git a/tests/examples/duckdb/signature_model.parquet b/tests/examples/duckdb/instrument/signature_model.parquet similarity index 100% rename from tests/examples/duckdb/signature_model.parquet rename to tests/examples/duckdb/instrument/signature_model.parquet diff --git a/tests/examples/duckdb/spec.csv b/tests/examples/duckdb/instrument/spec.csv similarity index 100% rename from tests/examples/duckdb/spec.csv rename to tests/examples/duckdb/instrument/spec.csv diff --git a/tests/examples/duckdb/string.parquet b/tests/examples/duckdb/instrument/string.parquet similarity index 100% rename from tests/examples/duckdb/string.parquet rename to tests/examples/duckdb/instrument/string.parquet diff --git a/tests/examples/duckdb/measurement/type=1/third_value=no/data_0.parquet b/tests/examples/duckdb/measurement/type=1/third_value=no/data_0.parquet new file mode 100644 index 00000000..0973fef9 Binary files /dev/null and b/tests/examples/duckdb/measurement/type=1/third_value=no/data_0.parquet differ diff --git a/tests/examples/duckdb/measurement/type=1/third_value=yes/data_0.parquet b/tests/examples/duckdb/measurement/type=1/third_value=yes/data_0.parquet new file mode 100644 index 00000000..3e85dc1d Binary files /dev/null and b/tests/examples/duckdb/measurement/type=1/third_value=yes/data_0.parquet differ diff --git a/tests/examples/duckdb/measurement/type=2/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet b/tests/examples/duckdb/measurement/type=2/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet new file mode 100644 index 00000000..670f3802 Binary files /dev/null and b/tests/examples/duckdb/measurement/type=2/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet differ diff --git a/tests/examples/duckdb/measurement/type=3/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet b/tests/examples/duckdb/measurement/type=3/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet new file mode 100644 index 00000000..c1d749f3 Binary files /dev/null and b/tests/examples/duckdb/measurement/type=3/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet differ diff --git a/tests/examples/duckdb/measurement/type=4/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet b/tests/examples/duckdb/measurement/type=4/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet new file mode 100644 index 00000000..3872d227 Binary files /dev/null and b/tests/examples/duckdb/measurement/type=4/third_value=__HIVE_DEFAULT_PARTITION__/data_0.parquet differ diff --git a/tests/examples/duckdb/measurement/type=5/third_value=fish/data_0.parquet b/tests/examples/duckdb/measurement/type=5/third_value=fish/data_0.parquet new file mode 100644 index 00000000..0152aaf6 Binary files /dev/null and b/tests/examples/duckdb/measurement/type=5/third_value=fish/data_0.parquet differ diff --git a/tests/examples/duckdb/measurement/type=5/third_value=fowl/data_0.parquet b/tests/examples/duckdb/measurement/type=5/third_value=fowl/data_0.parquet new file mode 100644 index 00000000..b4f4660b Binary files /dev/null and b/tests/examples/duckdb/measurement/type=5/third_value=fowl/data_0.parquet differ diff --git a/tests/test_dump.py b/tests/test_dump.py index d6a838a1..55811890 100644 --- a/tests/test_dump.py +++ b/tests/test_dump.py @@ -101,7 +101,7 @@ class EndToEndParquetTestCase(DatafakerTestCase): """Read in parquet, make some generators, output parquet.""" database_type = DuckTestDb - examples_dir = Path("examples/duckdb") + examples_dir = Path("examples/duckdb/instrument") def set_working_dir(self) -> None: """Change to our working directory.""" @@ -114,9 +114,10 @@ def setUp(self) -> None: super().setUp() # Grab all the files self.parquet_dir = Path(tempfile.mkdtemp("parq")) - example_dir = self.get_abs_example_dir() - for fname in os.listdir(example_dir): - shutil.copy(example_dir / fname, self.parquet_dir / fname) + example_dir = Path(self.get_abs_example_dir()) + for child in example_dir.iterdir(): + if child.is_file(): + shutil.copy(example_dir / child, self.parquet_dir / child.name) self.start_dir = os.getcwd() self.set_working_dir() @@ -133,22 +134,31 @@ def make_orm_yaml(self, runner: CliRunner) -> None: "make-tables", "--parquet-dir", str(self.parquet_dir), - "--orm-file", - "orm_auto.yaml", ], ) self.assertSuccess(result) - # Fix up the orm.yaml; the dates might not have types set - with Path("orm_auto.yaml").open(encoding="utf-8") as orm_fh: + with Path("orm.yaml").open(encoding="utf-8") as orm_fh: orm = yaml.load(orm_fh, yaml.SafeLoader) + # Let's check some of the types, primary and foreign keys t = orm["tables"] + self.assertEqual( + t["signature_model.parquet"]["columns"]["player_id"]["type"], "INTEGER" + ) + self.assertTrue(t["manufacturer.parquet"]["columns"]["id"]["primary"]) + self.assertFalse( + t["manufacturer.parquet"]["columns"]["name"].get("primary", False) + ) + self.assertEqual( + t["signature_model.parquet"]["columns"]["player_id"]["foreign_keys"], + ["player.parquet.id"], + ) + self.assertEqual( + t["signature_model.parquet"]["columns"]["based_on"]["type"], "INTEGER" + ) + # DATETIMEs cannot be detected t["manufacturer.parquet"]["columns"]["founded"]["type"] = "DATETIME" t["model.parquet"]["columns"]["introduced"]["type"] = "DATETIME" - t["signature_model.parquet"]["columns"]["player_id"]["type"] = "INTEGER" - t["signature_model.parquet"]["columns"]["player_id"]["foreign_keys"] = [ - "player.parquet.id" - ] - t["signature_model.parquet"]["columns"]["based_on"]["type"] = "INTEGER" + # "based_on" doesn't give the foreign key guesser anything to go on t["signature_model.parquet"]["columns"]["based_on"]["foreign_keys"] = [ "model.parquet.id" ] diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index dbf0182a..64c033f5 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -1,6 +1,8 @@ """Unit tests for datafaker.evaluators.proposal_ranking.""" from typing import Any +from sqlalchemy import Column, MetaData, Table, Text + from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation from datafaker.evaluators.proposal_ranking import ( ProposalRanking, @@ -227,7 +229,8 @@ def test_keyword_boost_can_flip_the_recommendation(self) -> None: def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" - choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + table = Table("t", MetaData(), Column("c", Text())) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) results = [ make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), make_result(overall_score=0.5, copy_fraction=0.0), @@ -307,7 +310,8 @@ def test_combined_score_is_clamped_to_one(self) -> None: def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: """A heavily-discounted best-fit candidate triggers the fallback warning.""" - choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + table = Table("t", MetaData(), Column("c", Text())) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) results = [ # Fits almost perfectly but is a resampler on a near-unique column: # gets discounted to (near) zero. diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py index dbd6415f..89191868 100644 --- a/tests/test_functional_mssql.py +++ b/tests/test_functional_mssql.py @@ -17,6 +17,7 @@ from tempfile import mkstemp import yaml +from sqlalchemy import Column, MetaData, Table, Text from sqlalchemy import create_engine as sa_create_engine from sqlalchemy import text from sqlalchemy.dialects import mssql as mssql_dialect # noqa: PLC0415 @@ -52,7 +53,9 @@ def setUp(self) -> None: (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as fh: fh.write( - make_tables_file(self.dsn, self.schema_name, engine=self.sync_engine) + make_tables_file( + self.dsn, self.schema_name, False, engine=self.sync_engine + ) ) def tearDown(self) -> None: @@ -119,9 +122,9 @@ def test_dialect_rand(self) -> None: """ChoiceProposer compiles its query with NEWID() not RANDOM() for mssql.""" dialect = mssql_dialect.dialect() + table = Table("manufacturer", MetaData(), Column("name", Text())) proposer = ZipfChoiceProposer( - table_name="manufacturer", - column_name="name", + column=table.c.name, values=["Blender", "Gibbs"], counts=[5, 5], sample_count=2, diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py index ec5e1744..99261c12 100644 --- a/tests/test_generators_dialect.py +++ b/tests/test_generators_dialect.py @@ -4,14 +4,19 @@ import unittest.mock from unittest.mock import MagicMock -from sqlalchemy import Column, Integer, MetaData, Select, Table, literal_column +from sqlalchemy import Column, Integer, MetaData, Select, Table, Text, literal_column from sqlalchemy.dialects import mssql, postgresql from sqlalchemy.types import DateTime from datafaker.dialects import SecondsDifference from datafaker.interactive.generators import get_aggregate_query from datafaker.interactive.missingness import MissingnessType -from datafaker.proposers.base import Buckets, PredefinedProposer, Proposer +from datafaker.proposers.base import ( + Buckets, + PredefinedProposer, + Proposer, + RelatedColumn, +) from datafaker.proposers.choice import ChoiceProposerFactory, ZipfChoiceProposer from datafaker.proposers.continuous import ( ContinuousLogDistributionProposerFactory, @@ -141,9 +146,13 @@ class TestChoiceGeneratorStoredQuery(DatafakerTestCase): """ChoiceGenerator._query is compiled to dialect-correct SQL at construction time.""" def _make_gen(self, dialect, sample_count=None, suppress_count=0): + table = Table( + "patient", + MetaData(), + Column("gender", Text()), + ) return ZipfChoiceProposer( - table_name="patient", - column_name="gender", + column=table.c.gender, values=["M", "F"], counts=[70, 30], sample_count=sample_count, @@ -592,7 +601,7 @@ def test_date_after_proposer_stddev_clause_mssql(self) -> None: sd=1.0, mean=86400.0, column=tbl.c.end_date, - anchor=tbl.c.start_date, + anchor_related=RelatedColumn(tbl.c.start_date, None), engine=engine, ) clauses = proposer.select_aggregate_clauses() @@ -620,7 +629,7 @@ def test_date_after_proposer_stddev_clause_postgresql(self) -> None: sd=1.0, mean=86400.0, column=tbl.c.end_date, - anchor=tbl.c.start_date, + anchor_related=RelatedColumn(tbl.c.start_date, None), engine=engine, ) clauses = proposer.select_aggregate_clauses() diff --git a/tests/test_interactive_copy_generators.py b/tests/test_interactive_copy_generators.py new file mode 100644 index 00000000..b441533d --- /dev/null +++ b/tests/test_interactive_copy_generators.py @@ -0,0 +1,262 @@ +""" Tests for the configure-generators command. """ +import re +from collections.abc import MutableMapping +from typing import Any, Iterable + +from sqlalchemy import select +from sqlalchemy.orm import aliased + +from datafaker.interactive.base import DbCmd +from tests.test_interactive_generators import MockGeneratorCmd +from tests.utils import GeneratesDBTestCase, MsSqlTestDb + + +class ConfigureCopyGeneratorsWithDateTests(GeneratesDBTestCase): + """Test `configure-generators` with the `instrument.sql` database.""" + + dump_file_path = "date.sql" + database_name = "date_tables" + schema_name = "public" + use_temporary_cwd = True + + def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: + """Get the command we are using for this test case.""" + return MockGeneratorCmd( + DbCmd.Settings(self.dsn, self.schema_name, config, self.metadata, None) + ) + + def test_same_table_copy(self) -> None: + """Test that the same-table copy is proposed and compared.""" + config = { + "tables": { + "happening": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = "generic.anchored_provider.copy [from name]" + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.reset() + gc.do_compare(str(prop[0])) + self.assertEqual(gc.messages[0][0], gc.NOT_PRIVATE_TEXT) + self.assertEqual(gc.messages[1][0], gc.REQUIRES_NO_SOURCE_DATA_TEXT) + + def test_cross_table_copy(self) -> None: + """Test that the cross-table copy is proposed and compared.""" + config = { + "tables": { + "person": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + provider_name2 = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via other_person_id)]" + ) + self.assertIn(provider_name2, proposals.keys()) + prop2 = proposals[provider_name2] + gc.reset() + gc.do_compare(f"{prop[0]} {prop2[0]}") + self.assertEqual(gc.messages[0][0], gc.NOT_PRIVATE_TEXT) + self.assertEqual(gc.messages[1][0], gc.REQUIRES_NO_SOURCE_DATA_TEXT) + + def test_cross_table_copy_actually_copies(self) -> None: + """Test that the cross-table copy is copies.""" + config = { + "tables": { + "person": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.name") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.do_set(str(prop[0])) + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via other_person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.do_set(str(prop[0])) + gc.do_quit("") + self.generate_data(gc.config, num_passes=50) + assert self.dst_engine is not None + with self.dst_engine.connect() as conn: + pert = self.dst_metadata.tables["person"] + other_pert = aliased(pert) + hapt = self.dst_metadata.tables["happening"] + query = ( + select( + hapt.columns["name"].label("hn"), + hapt.columns["comment"].label("hc"), + pert.columns["name"].label("pn"), + other_pert.columns["name"].label("opn"), + ) + .select_from(hapt) + .join( + pert, + onclause=pert.columns["id"] == hapt.columns["person_id"], + ) + .join( + other_pert, + onclause=other_pert.columns["id"] + == hapt.columns["other_person_id"], + isouter=True, + ) + ) + dst_result = conn.execute(query).fetchall() + saw_other = False + for row in dst_result: + if row.opn: + saw_other = True + self.assertEqual(row.hc, row.opn) + self.assertEqual(row.hn, row.pn) + self.assertTrue(saw_other) + + def test_cross_table_fk_copy(self) -> None: + """Test that the cross-table foreign key copy is proposed.""" + config = { + "tables": { + "person": { + "columns": { + "parent_of": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from parent_of of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + + COPY_FROM_RE = re.compile(r"generic.anchored_provider.copy \[from ([^\]]+)\]") + + def get_same_table_copy_proposals(self, proposals: Iterable[str]) -> set[str]: + """Return a set of columns proposed to copy from.""" + return { + p.group(1) + for p in (self.COPY_FROM_RE.match(prop) for prop in proposals) + if p is not None + } + + def test_different_target_fk_copy_is_not_proposed(self) -> None: + """Test a foreign key copy with a different target table is not proposed.""" + config = { + "tables": { + "happening": { + "columns": { + "previous_happening_id": { + "roles": ["source"], + }, + "person_id": { + "roles": ["source"], + }, + } + } + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = "generic.anchored_provider.copy [from person_id]" + self.assertIn(provider_name, proposals.keys()) + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertIn("person_id", all_copies) + self.assertNotIn("previous_happening_id", all_copies) + + def test_nullable_copy_nonnullable_is_not_proposed(self) -> None: + """Test a copy from a nullable column to a nonnullable column is not proposed.""" + config = { + "tables": { + "happening": { + "columns": { + "other_person_id": { + "roles": ["source"], + }, + "person_id": { + "roles": ["source"], + }, + } + } + } + } + with self._get_cmd(config) as gc: + # person -> other_person is allowed + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertIn("person_id", all_copies) + # other_person -> person is allowed + gc.do_next("happening.person_id") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertNotIn("other_person_id", all_copies) + + +# Note that this test won't work with DuckDB because it needs foreign keys to work +class ConfigureCopyGeneratorsWithDateMsSqlTests(ConfigureCopyGeneratorsWithDateTests): + """Test `configure-generators` with `instrument.sql` with MS SQL.""" + + database_type = MsSqlTestDb + schema_name = None diff --git a/tests/test_interactive_date_generators.py b/tests/test_interactive_date_generators.py index 5dd18f02..9a0be810 100644 --- a/tests/test_interactive_date_generators.py +++ b/tests/test_interactive_date_generators.py @@ -171,7 +171,8 @@ def test_cross_table_interval_end_to_end(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via manufacturer_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] @@ -221,7 +222,7 @@ class ConfigureGeneratorsWithInstrumentsMsSqlTests( class ConfigureGeneratorsWithDateTests(GeneratesDBTestCase): - """Test `configure-generators` with the `instrument.sql` database.""" + """Test `configure-generators` with the `date.sql` database.""" dump_file_path = "date.sql" database_name = "date_tables" @@ -281,13 +282,15 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] provider_name2 = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor2} of table {atable}]" + f" [anchored to {anchor2} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name2, proposals.keys()) prop2 = proposals[provider_name2] @@ -302,13 +305,15 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] provider_name2 = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor2} of table {atable}]" + f" [anchored to {anchor2} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name2, proposals.keys()) prop2 = proposals[provider_name2] @@ -318,10 +323,35 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: self.assertEqual(gc.messages[1][0], gc.REQUIRES_SOURCE_DATA_TEXT) self.assertEqual(gc.messages[2][0], gc.PROVIDING_VALUES_TEXT) + def test_regression_date_start_crash(self) -> None: + """Test for regression where ``start`` role on a date leads to crash.""" + config = { + "tables": { + "person": { + "columns": { + "date_of_birth": { + "roles": ["start"], + }, + }, + }, + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.at_date") + gc.reset() + gc.do_propose("") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.normal_date_fk" + " [anchored to date_of_birth of table person" + " (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + # Note that this test won't work with DuckDB because it needs foreign keys to work class ConfigureGeneratorsWithDateMsSqlTests(ConfigureGeneratorsWithDateTests): - """Test `configure-generators` with `instrument.sql` with MS SQL.""" + """Test `configure-generators` with `date.sql` with MS SQL.""" database_type = MsSqlTestDb schema_name = None diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 46b20c30..678d1237 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -240,9 +240,9 @@ def test_set_generator_choice(self) -> None: gc.config["src-stats"][0]["query"], ( "SELECT _counted.value \n" - f'FROM (SELECT "{column}" AS value, count("{column}") AS count \n' + f"FROM (SELECT {table}.{column} AS value, count({table}.{column}) AS count \n" f"FROM {table} \n" - f'WHERE "{column}" IS NOT NULL GROUP BY "{column}") ' + f"WHERE {table}.{column} IS NOT NULL GROUP BY {table}.{column}) " "AS _counted ORDER BY _counted.count DESC" ), ) diff --git a/tests/test_interactive_missingness.py b/tests/test_interactive_missingness.py index c1da7c54..747f9e0f 100644 --- a/tests/test_interactive_missingness.py +++ b/tests/test_interactive_missingness.py @@ -1,7 +1,8 @@ """ Tests for the configure-missingness command. """ import random from collections.abc import MutableMapping -from typing import Any +from pathlib import Path +from typing import Any, Iterable from sqlalchemy import select @@ -10,6 +11,15 @@ from tests.utils import GeneratesDBTestCase, RequiresDBTestCase, TestDbCmdMixin +def count_nonnull(it: Iterable): + """Count the number of non-null values in an iterable.""" + n = 0 + for v in it: + if v is not None: + n += 1 + return n + + class MockMissingnessCmd(MissingnessCmd, TestDbCmdMixin): """MissingnessCmd but mocked""" @@ -107,3 +117,39 @@ def test_create_with_missingness(self) -> None: patterns.add(p + b) # all pattern possibilities should be present self.assertSetEqual(patterns, {0, 1, 2, 3}) + + +class ConfigureMissingnessEndToEndTests(ConfigureMissingnessTestsWithGeneration): + """Test `configure-missing` end-to-end.""" + + use_temporary_cwd = True + copy_files = ["row_generators.py", "story_generators.py"] + copy_from_directory = Path("examples") + + def test_missingness_end_to_end(self) -> None: + """Test that if an interval end is applicable it gets proposed and works.""" + table = "signature_model" + with self._get_cmd({}) as mc: + mc.do_next(table) + mc.do_sampled("") + mc.reset() + mc.do_quit("") + self.generate_data(mc.config, num_passes=100) + self.assertIs( + type( + mc.config["tables"][table]["missingness_generators"][0]["columns"][ + 0 + ] + ), + str, + ) + assert self.dst_engine is not None + with self.dst_engine.connect() as conn: + stmt = select(self.metadata.tables[table]) + rows = conn.execute(stmt).mappings().fetchall() + name_count = count_nonnull(r.name for r in rows) + player_id_count = count_nonnull(r.player_id for r in rows) + based_on_count = count_nonnull(r.based_on for r in rows) + self.assertEqual(name_count, 100) + self.assertAlmostEqual(player_id_count, 50, delta=20) + self.assertAlmostEqual(based_on_count, 70, delta=20) diff --git a/tests/test_main.py b/tests/test_main.py index 02878b14..d84c833e 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -111,7 +111,11 @@ def test_make_tables( ) mock_make_tables_file.assert_called_once_with( - "postgresql://suser:spassword@shost:5432/sdbname", None, None + "postgresql://suser:spassword@shost:5432/sdbname", + None, + views_to_tables=False, + parquet_dir=None, + guess_keys=False, ) mock_path.return_value.write_text.assert_called_once_with( "some text", encoding="utf-8" @@ -176,6 +180,8 @@ def test_make_tables_with_force_enabled( for force_option in ["--force", "-f"]: with self.subTest(f"Using option {force_option}"): + mock_make_tables.reset_mock() + mock_path.reset_mock() result: Result = runner.invoke( app, [ @@ -188,16 +194,15 @@ def test_make_tables_with_force_enabled( mock_make_tables.assert_called_once_with( mock_get_settings.return_value.src_dsn, mock_get_settings.return_value.src_schema, - None, + views_to_tables=False, + parquet_dir=None, + guess_keys=False, ) mock_path.return_value.write_text.assert_called_once_with( mock_tables_output, encoding="utf-8" ) self.assertSuccess(result) - mock_make_tables.reset_mock() - mock_path.reset_mock() - @patch("datafaker.main.Path") @patch("datafaker.settings.get_settings") def test_incorrect_dialect_causes_nice_error_message( diff --git a/tests/test_make.py b/tests/test_make.py index 9327ea6a..cad6eeec 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -3,6 +3,7 @@ import tempfile from pathlib import Path from typing import Any +from unittest import SkipTest from unittest.mock import MagicMock, patch import pandas as pd @@ -20,12 +21,107 @@ from sqlalchemy.dialects.mysql.types import INTEGER from sqlalchemy.dialects.postgresql import UUID +from datafaker.db_utils import create_db_engine, get_sync_engine from datafaker.make import ( _get_default_generator, _get_provider_for_column, make_src_stats, + make_tables_file, ) -from tests.utils import DatafakerTestCase, GeneratesDBTestCase, RequiresDBTestCase +from tests.utils import ( + DatafakerTestCase, + GeneratesDBTestCase, + MsSqlTestDb, + PostgresTestDb, + RequiresDBTestCase, +) + + +class TestSerialize(DatafakerTestCase): + """Unit tests for serialize_metadata.""" + + database_type = PostgresTestDb + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + skip_msg = cls.database_type.skip() + if skip_msg: + raise SkipTest(skip_msg) + cls.database_type.setup() + + def setUp(self) -> None: + super().setUp() + self.database = self.database_type() + self.database.open() + self.database.run_sql(self.get_abs_example_dir() / "date.sql") + self.engine = create_db_engine(self.database.get_dsn("date_tables"), None) + + def test_metadata_to_dict(self) -> None: + """Test metadata_to_dict.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=False, + parquet_dir=None, + engine=get_sync_engine(self.engine), + guess_keys=False, + ) + orm = yaml.load(orm_yaml, yaml.Loader) + table_names = set(orm["tables"].keys()) + self.assert_subset({"person", "happening"}, table_names) + self.assertNotIn("happening_view", table_names) + happening_columns = orm["tables"]["happening"]["columns"] + happening_column_names = set(happening_columns.keys()) + self.assert_subset( + {"id", "name", "comment", "person_id"}, + happening_column_names, + ) + self.assertEqual(happening_columns["person_id"]["type"], "INTEGER") + self.assertFalse(happening_columns["name"]["primary"]) + self.assertTrue(happening_columns["id"]["primary"]) + + def test_views_to_dict(self) -> None: + """Test views_to_tables.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=True, + parquet_dir=None, + engine=get_sync_engine(self.engine), + ) + orm = yaml.load(orm_yaml, yaml.Loader) + table_names = set(orm["tables"].keys()) + self.assert_subset({"person", "happening", "happening_view"}, table_names) + happening_columns = orm["tables"]["happening_view"]["columns"] + happening_column_names = set(happening_columns.keys()) + self.assert_subset( + {"id", "name", "person_id"}, + happening_column_names, + ) + self.assertEqual(happening_columns["person_id"]["type"], "INTEGER") + self.assertFalse(happening_columns["name"]["primary"]) + + def test_views_to_dict_guess_keys(self) -> None: + """Test views_to_tables with foreign keys and primary keys guessed.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=True, + parquet_dir=None, + engine=get_sync_engine(self.engine), + guess_keys=True, + ) + orm = yaml.load(orm_yaml, yaml.Loader) + view_columns = orm["tables"]["happening_view"]["columns"] + self.assertTrue(view_columns["id"]["primary"]) + self.assertEqual(view_columns["person_id"]["foreign_keys"], ["person.id"]) + + +class TestSerializeMsSql(TestSerialize): + """Unit tests for serialize_metadata with an MS SQL database.""" + + database_type = MsSqlTestDb class TestGetDefaultGenerator(DatafakerTestCase): diff --git a/tests/test_noninteractive_generators.py b/tests/test_noninteractive_generators.py index 9dee1166..2c994da4 100644 --- a/tests/test_noninteractive_generators.py +++ b/tests/test_noninteractive_generators.py @@ -7,7 +7,7 @@ from datafaker.interactive import update_config_generators from datafaker.interactive.base import DbCmd from datafaker.utils import get_property -from tests.utils import RequiresDBTestCase +from tests.utils import DuckTestDb, RequiresDBTestCase class NonInteractiveTests(RequiresDBTestCase): @@ -20,9 +20,14 @@ class NonInteractiveTests(RequiresDBTestCase): schema_name = "public" @patch("datafaker.interactive.Path") - @patch( - "datafaker.interactive.csv.reader", - return_value=iter( + @patch("datafaker.interactive.csv.reader") + def test_non_interactive_configure_generators( + self, mock_csv_reader: MagicMock, _mock_path: MagicMock + ) -> None: + """ + test that we can set generators from a CSV file + """ + mock_csv_reader.return_value = iter( [ ["observation", "type", "dist_gen.weighted_choice [sampled]"], [ @@ -40,14 +45,7 @@ class NonInteractiveTests(RequiresDBTestCase): ], ["observation", "third_value", "dist_gen.weighted_choice"], ] - ), - ) - def test_non_interactive_configure_generators( - self, _mock_csv_reader: MagicMock, _mock_path: MagicMock - ) -> None: - """ - test that we can set generators from a CSV file - """ + ) config: MutableMapping[str, Any] = {} spec_csv = MagicMock() spec_csv.open.return_value.__enter__.return_value = "mock spec.csv file" @@ -76,9 +74,14 @@ def test_non_interactive_configure_generators( ) @patch("datafaker.interactive.Path") - @patch( - "datafaker.interactive.csv.reader", - return_value=iter( + @patch("datafaker.interactive.csv.reader") + def test_non_interactive_configure_null_partitioned( + self, mock_csv_reader: MagicMock, _mock_path: MagicMock + ) -> None: + """ + test that we can set multi-column generators from a CSV file + """ + mock_csv_reader.return_value = iter( [ [ "observation", @@ -86,14 +89,7 @@ def test_non_interactive_configure_generators( "null-partitioned grouped_multivariate_lognormal", ], ] - ), - ) - def test_non_interactive_configure_null_partitioned( - self, _mock_csv_reader: MagicMock, _mock_path: MagicMock - ) -> None: - """ - test that we can set multi-column generators from a CSV file - """ + ) config: MutableMapping[str, Any] = {} spec_csv = MagicMock() spec_csv.open.return_value.__enter__.return_value = "mock spec.csv file" @@ -132,26 +128,24 @@ def test_non_interactive_configure_null_partitioned( ) @patch("datafaker.interactive.Path") - @patch( - "datafaker.interactive.csv.reader", - return_value=iter( - [ - [ - "observation", - "type first_value second_value third_value", - "null-partitioned grouped_multivariate_lognormal", - ], - ] - ), - ) + @patch("datafaker.interactive.csv.reader") def test_non_interactive_configure_null_partitioned_where_existing_merges( - self, _mock_csv_reader: MagicMock, _mock_path: MagicMock + self, mock_csv_reader: MagicMock, _mock_path: MagicMock ) -> None: """ test that we can set multi-column generators from a CSV file, but where there are already multi-column generators configured that will have to be unmerged. """ + mock_csv_reader.return_value = iter( + [ + [ + "observation", + "type first_value second_value third_value", + "null-partitioned grouped_multivariate_lognormal", + ], + ] + ) config = { "tables": { "observation": { @@ -225,3 +219,9 @@ def test_non_interactive_configure_null_partitioned_where_existing_merges( ), '"grouped_multivariate_lognormal"', ) + + +class NonInteractiveTestsDuckDB(NonInteractiveTests): + """DuckDB version of NonInteractiveTests.""" + + database_type = DuckTestDb diff --git a/tests/test_parquet2orm.py b/tests/test_parquet2orm.py index f4564a34..b09770dc 100644 --- a/tests/test_parquet2orm.py +++ b/tests/test_parquet2orm.py @@ -6,15 +6,14 @@ from collections.abc import Iterable from pathlib import Path from typing import Any -from unittest import TestCase from unittest.mock import MagicMock, patch import pandas as pd from sqlalchemy import text -from typer import Exit from datafaker.db_utils import create_db_engine, get_sync_engine -from datafaker.parquet2orm import get_parquet_orm +from datafaker.parquet2orm import add_guessed_keys, get_initial_parquet_orm +from tests.utils import DatafakerTestCase class HasValues: @@ -37,7 +36,7 @@ def __repr__(self) -> str: return f"HasValues<{self.value_set}>" -class ParquetDirTestCase(TestCase): +class ParquetDirTestCase(DatafakerTestCase): """Base class providing a temporary directory for parquet files.""" def setUp(self) -> None: @@ -83,8 +82,9 @@ def test_can_infer_column_type(self) -> None: } } self.write_parquet(data) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertSetEqual(set(orm.keys()), set(data.keys())) self.assertIn("columns", orm["fruit.parquet"]) self.assertSetEqual( @@ -113,8 +113,9 @@ def test_can_infer_primary_key(self, mock_logger: MagicMock) -> None: } } self.write_parquet(data) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertSetEqual(set(orm.keys()), set(data.keys())) primary_keys = { name @@ -126,7 +127,7 @@ def test_can_infer_primary_key(self, mock_logger: MagicMock) -> None: mock_logger.warning.assert_called_once_with( "Found multiple likely primary keys for table %s: %s", "fruit.parquet", - HasValues({"fruit_id", "FruitKey"}), + "FruitKey, fruit_id", ) def test_can_infer_foreign_key(self) -> None: @@ -140,8 +141,9 @@ def test_can_infer_foreign_key(self) -> None: "seed.parquet": {"seed_id": [1, 2], "name": ["orange pip", "grape seed"]}, } self.write_parquet(data) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertSetEqual(set(orm.keys()), set(data.keys())) self.assertNotIn("foreign_keys", orm["fruit.parquet"]["columns"]["fruit_id"]) self.assertNotIn("foreign_keys", orm["fruit.parquet"]["columns"]["name"]) @@ -177,8 +179,9 @@ def test_partitioned_directory_is_one_table(self) -> None: "visit/yr=__MISSING__/p00001.parquet": {"visit_id": [3], "ward": ["c"]}, } ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertSetEqual(set(orm.keys()), {"visit"}) cols = orm["visit"]["columns"] self.assertSetEqual(set(cols.keys()), {"visit_id", "ward", "yr"}) @@ -189,7 +192,9 @@ def test_partitioned_directory_is_one_table(self) -> None: def test_partition_key_type_is_not_a_warning(self, mock_logger: MagicMock) -> None: """Test the partition key does not log 'Could not determine type'.""" self.write_files({"visit/yr=2023/p1.parquet": {"visit_id": [1]}}) - get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) + assert orm is not None + add_guessed_keys(orm) mock_logger.warning.assert_not_called() def test_same_part_names_in_different_tables_do_not_collide(self) -> None: @@ -200,7 +205,9 @@ def test_same_part_names_in_different_tables_do_not_collide(self) -> None: "drug/yr=2023/p00001.parquet": {"drug_id": [1], "dose": [1.5]}, } ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) + assert orm is not None + add_guessed_keys(orm) assert orm is not None self.assertSetEqual(set(orm.keys()), {"visit", "drug"}) self.assertSetEqual( @@ -208,7 +215,8 @@ def test_same_part_names_in_different_tables_do_not_collide(self) -> None: ) self.assertSetEqual(set(orm["visit"]["columns"].keys()), {"visit_id", "yr"}) - def test_files_and_directories_mix(self) -> None: + @patch("datafaker.parquet2orm.logger") + def test_files_and_directories_mix(self, mock_logger: MagicMock) -> None: """Test a mixture of loose parquet files and parquet dirs are read as separate tables. Checks empty directory are not treated as partitioned tables.""" self.write_files( @@ -219,9 +227,11 @@ def test_files_and_directories_mix(self) -> None: ) (self.parquet_dir / "empty").mkdir() - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertSetEqual(set(orm.keys()), {"fruit.parquet", "visit"}) + mock_logger.warning.assert_called_once_with("Entry %s is empty", "empty") def test_nested_plain_directories_are_one_table(self) -> None: """Test a subdirectory of unpartitioned files is one table.""" @@ -231,34 +241,43 @@ def test_nested_plain_directories_are_one_table(self) -> None: "sub/b.parquet": {"x": [2]}, } ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertSetEqual(set(orm.keys()), {"sub"}) def test_parquet_dir_that_is_a_dataset(self) -> None: """Test the directory given can itself be a partitioned table.""" self.write_files( { - "yr=2022/p1.parquet": {"visit_id": [1]}, - "yr=2023/p1.parquet": {"visit_id": [2]}, + "visit/yr=2022/p1.parquet": {"visit_id": [1]}, + "visit/yr=2023/p1.parquet": {"visit_id": [2]}, } ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None - self.assertSetEqual(set(orm.keys()), {self.parquet_dir.resolve().name}) + add_guessed_keys(orm) + self.assertSetEqual(set(orm.keys()), {"visit"}) only = next(iter(orm.values())) self.assertSetEqual(set(only["columns"].keys()), {"visit_id", "yr"}) - def test_parquet_dir_dataset_must_be_consistent(self) -> None: + @patch("datafaker.parquet2orm.logger") + def test_parquet_dir_dataset_must_be_consistent( + self, mock_logger: MagicMock + ) -> None: """Test a directory of unrelated hive-named tables is still rejected.""" self.write_files( { - "yr=2022/p1.parquet": {"visit_id": [1]}, - "yr=2023/p1.parquet": {"other": ["x"]}, + "visit/yr=2022/p1.parquet": {"visit_id": [1]}, + "visit/yr=2023/p1.parquet": {"other": ["x"]}, } ) - with self.assertRaises(Exit): - get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) + assert orm is not None + self.assertDictEqual(orm, {}) + mock_logger.error.assert_called_once_with( + "Entry %s cannot be read as parquet", "visit" + ) @patch("datafaker.parquet2orm.logger") def test_differing_columns_are_rejected(self, mock_logger: MagicMock) -> None: @@ -269,20 +288,16 @@ def test_differing_columns_are_rejected(self, mock_logger: MagicMock) -> None: "sub/b.parquet": {"drug_id": [1]}, } ) - with self.assertRaises(Exit): - get_parquet_orm(self.parquet_dir) - mock_logger.error.assert_called_once() - message = ( - mock_logger.error.call_args.args[0] % mock_logger.error.call_args.args[1:] + orm = get_initial_parquet_orm(self.parquet_dir) + assert orm is not None + self.assertDictEqual(orm, {}) + mock_logger.error.assert_called_once_with( + "Entry %s cannot be read as parquet", "sub" ) - self.assertIn("sub", message) - self.assertIn("missing columns ['visit_id', 'ward']", message) - self.assertIn("extra columns ['drug_id']", message) - self.assertIn("move each table into its own directory", message) @patch("datafaker.parquet2orm.logger") - def test_differing_types_are_a_warning(self, mock_logger: MagicMock) -> None: - """Test the same column with a different type warns but is accepted.""" + def test_differing_types_is_disallowed(self, mock_logger: MagicMock) -> None: + """Test the same column with a different type causes an error.""" self.write_files( { "visit/yr=2022/p1.parquet": {"visit_id": [1]}, @@ -290,14 +305,12 @@ def test_differing_types_are_a_warning(self, mock_logger: MagicMock) -> None: "visit/yr=2023/p2.parquet": {"visit_id": [2.5]}, } ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None - self.assertEqual(orm["visit"]["columns"]["visit_id"]["type"], "INTEGER") - mock_logger.error.assert_not_called() - mock_logger.warning.assert_called_once() - args = mock_logger.warning.call_args.args - self.assertEqual(args[1:3], ("visit", "visit_id")) - self.assertEqual(args[3], ["float64", "int64"]) + self.assertDictEqual(orm, {}) + mock_logger.error.assert_called_once_with( + "Entry %s cannot be read as parquet", "visit" + ) @patch("datafaker.parquet2orm.logger") def test_int32_and_nullable_int32_are_the_same_type( @@ -316,11 +329,14 @@ def test_int32_and_nullable_int32_are_the_same_type( pd.DataFrame({"age": pd.array([None, 2], dtype="Int32")}).to_parquet( self.parquet_dir / "person/yr=2023/p1.parquet" ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertEqual(orm["person"]["columns"]["age"]["type"], "INTEGER") - for call in mock_logger.warning.call_args_list: - self.assertNotIn("different types", call.args[0]) + mock_logger.warning.assert_called_once_with( + "Found no likely primary keys for table %s", + "person", + ) def test_nullable_int_first_file_is_still_integer(self) -> None: """Test a pandas ``Int32`` column is typed ``INTEGER``.""" @@ -328,23 +344,13 @@ def test_nullable_int_first_file_is_still_integer(self) -> None: pd.DataFrame({"age": pd.array([None, 2], dtype="Int32")}).to_parquet( self.parquet_dir / "person/yr=2022/p1.parquet" ) - orm = get_parquet_orm(self.parquet_dir) + orm = get_initial_parquet_orm(self.parquet_dir) assert orm is not None + add_guessed_keys(orm) self.assertEqual(orm["person"]["columns"]["age"]["type"], "INTEGER") - def test_differing_partition_keys_are_rejected(self) -> None: - """Test files partitioned differently are rejected.""" - self.write_files( - { - "visit/yr=2022/p1.parquet": {"visit_id": [1]}, - "visit/p2.parquet": {"visit_id": [2]}, - } - ) - with self.assertRaises(Exit): - get_parquet_orm(self.parquet_dir) - -class PartitionedParquetEngine(TestCase): +class PartitionedParquetEngine(DatafakerTestCase): """Integration tests for querying Parquet data through the parquet_dir option.""" def setUp(self) -> None: diff --git a/tests/test_utils_mssql.py b/tests/test_utils_mssql.py index 5e27cd8f..44748ed0 100644 --- a/tests/test_utils_mssql.py +++ b/tests/test_utils_mssql.py @@ -107,8 +107,9 @@ def test_duckdb_parquet_dir_sets_search_path(self) -> None: if len(calls[0].args) > 1 else calls[0].kwargs.get("settings", {}) ) - self.assertIn("file_search_path", settings_passed) - self.assertEqual(settings_passed["file_search_path"], f"'{parq_dir}'") + all_settings = "\n".join(settings_passed) + self.assertIn("file_search_path", all_settings) + self.assertIn(f"SET file_search_path TO '{parq_dir}';", all_settings) def test_mssql_dsn_schema_sets_translate_map(self) -> None: """schema_translate_map is set even for an MS-SQL DSN (engine creation, no connect).""" @@ -144,9 +145,11 @@ def test_reflect_called_with_schema(self) -> None: mock_meta_data.return_value = mock_md mock_md.reflect.return_value = None - get_metadata(mock_engine, schema_name="myschema") + get_metadata(mock_engine, schema_name="myschema", views_to_tables=False) - mock_md.reflect.assert_called_once_with(mock_engine, schema="myschema") + mock_md.reflect.assert_called_once_with( + mock_engine, schema="myschema", views=False + ) def test_reflect_called_without_schema_when_none(self) -> None: """get_metadata passes schema=None to reflect when no schema_name is given.""" @@ -158,6 +161,8 @@ def test_reflect_called_without_schema_when_none(self) -> None: mock_meta_data.return_value = mock_md mock_md.reflect.return_value = None - get_metadata(mock_engine) + get_metadata(mock_engine, schema_name=None, views_to_tables=False) - mock_md.reflect.assert_called_once_with(mock_engine, schema=None) + mock_md.reflect.assert_called_once_with( + mock_engine, schema=None, views=False + ) diff --git a/tests/utils.py b/tests/utils.py index 8bdf5057..6673d7c5 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -189,7 +189,7 @@ class DuckTestDb(TestDatabaseBase): SQL_REMOVALS_RE = re.compile( r"^(CREATE DATABASE .*;)|(\\.*)|(ALTER [^;]*;)|(\bUSING\s+btree)", re.MULTILINE ) - # Postgres' default stupid time format, ingoring time zone for now + # Postgres' default stupid time format, ignoring time zone for now TIME_FORMAT_RE = re.compile( r"'([A-Z][a-z]+ \d+ \d\d:\d\d:\d\d \d\d\d\d) ([A-Z+\-0-9:]+)'" ) @@ -284,7 +284,7 @@ class MsSqlTestDb(TestDatabaseBase): _EOL_RE = re.compile(r";$", re.MULTILINE) _TIMESTAMP_RE = re.compile(r"(TIMESTAMP)\s+WITH\s+TIME\s+ZONE", re.IGNORECASE) _ALTER_OWNER_RE = re.compile( - r'ALTER\s+(TABLE|DATABASE)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;' + r'ALTER\s+(TABLE|DATABASE|VIEW)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;' ) _ALTER_ONLY_RE = re.compile(r"ALTER\s+TABLE\s+ONLY\s+") _CREATE_INDEX_RE = re.compile( @@ -764,7 +764,7 @@ def setUp(self) -> None: # Generate the `orm.yaml` from the database (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as orm_fh: - orm_fh.write(make_tables_file(self.dsn, self.schema_name)) + orm_fh.write(make_tables_file(self.dsn, self.schema_name, False)) # Create a separate empty destination database self.make_destination_database("dst")