diff --git a/datafaker/db_utils.py b/datafaker/db_utils.py index cf48d4b6..6aa47622 100644 --- a/datafaker/db_utils.py +++ b/datafaker/db_utils.py @@ -11,15 +11,7 @@ # pylint: disable=no-name-in-module from psycopg2.errors import UndefinedObject # ty: ignore[unresolved-import] -from sqlalchemy import ( - Column, - Connection, - Engine, - ForeignKey, - create_engine, - event, - select, -) +from sqlalchemy import Connection, Engine, ForeignKey, create_engine, event, select from sqlalchemy.engine.interfaces import DBAPIConnection from sqlalchemy.exc import ( IntegrityError, @@ -169,22 +161,34 @@ def create_db_engine( logger.error("DSN %s is malformed: %s", db_dsn, exc) raise Exit(1) from exc - settings = {} + settings: list[str] = [] + dialect_name = get_sync_engine(engine).dialect.name if schema_name is not None: - if get_sync_engine(engine).dialect.name == "mssql": - engine = engine.execution_options(schema_translate_map={None: schema_name}) - else: - settings["search_path"] = schema_name + match dialect_name: + case "mssql": + engine = engine.execution_options( + schema_translate_map={None: schema_name} + ) + case "snowflake": + settings.append(f"USE SCHEMA {schema_name};") + case _: + settings.append(f"SET search_path TO {schema_name};") dataset_views: list[str] = [] if parquet_dir is not None: - tables = find_parquet_tables(parquet_dir) - joined = ",".join( - sorted({str(t.root.parent) for t in tables if not t.is_dataset}) - ) - # double up single quotes - dj = joined.replace("'", "''") - # enclose in single quotes - settings["file_search_path"] = f"'{dj}'" + match dialect_name: + case "duckdb": + tables = find_parquet_tables(parquet_dir) + joined = ",".join( + sorted({str(t.root.parent) for t in tables if not t.is_dataset}) + ) + # double up single quotes + dj = joined.replace("'", "''") + settings.append(f"SET file_search_path TO '{dj}';") + case _: + tables = [] + logger.error( + "--parquet-dir cannot be used with %s, only duckdb.", dialect_name + ) dataset_views = [_dataset_view_sql(t) for t in tables if t.is_dataset] if settings or dataset_views: @@ -227,11 +231,15 @@ def create_db_engine_dst( return create_db_engine(db_dsn, schema_name, use_asyncio) -def get_metadata(engine: Engine, schema_name: Optional[str] = None) -> MetaData: +def get_metadata( + engine: Engine, + schema_name: Optional[str], + views_to_tables: bool, +) -> MetaData: """Get the MetaData object associated with the engine passed.""" md = MetaData() try: - md.reflect(engine, schema=schema_name) + md.reflect(engine, schema=schema_name, views=views_to_tables) except OperationalError as exc: logger.error("Cannot connect to database: %s", exc) raise Exit(1) from exc @@ -267,16 +275,14 @@ def create_dataset_views( cursor.close() -def set_db_settings(connection: DBAPIConnection, settings: Mapping[str, str]) -> None: - """Set the SEARCH_PATH for a PostgreSQL connection.""" +def set_db_settings(connection: DBAPIConnection, settings: list[str]) -> None: + """Run commands.""" # https://docs.sqlalchemy.org/en/20/dialects/postgresql.html#remote-schema-table-introspection-and-postgresql-search-path existing_autocommit = connection.autocommit connection.autocommit = True cursor = connection.cursor() - # Parametrised queries don't work with asyncpg, hence the f-string. - sql = "".join(f"SET {k} TO {v};" for k, v in settings.items()) - cursor.execute(sql) + cursor.execute("\n".join(settings)) cursor.close() connection.autocommit = existing_autocommit @@ -299,25 +305,6 @@ def get_orm_metadata( return metadata -def get_fk_column_between( - from_: Table, to_: Table -) -> tuple[Column, ForeignKey] | tuple[None, None]: - """ - Get a foreign key between two tables. - - :param from_: The table on which the foreign key should be defined. - :param to_: The table to which the foreign key should refer. - :return: A pair (column on from_, foreign key to to_) if such a key exists, - or (None, None) if not. - """ - for c in from_.columns: - fk: ForeignKey - for fk in c.foreign_keys: - if fk.column.table == to_: - return (c, fk) - return (None, None) - - def is_numeric_sql_type(column_type: TypeEngine | None) -> bool: """Whether a SQLAlchemy column type is Numeric or Integer.""" return isinstance(column_type, (Numeric, Integer)) diff --git a/datafaker/dialects.py b/datafaker/dialects.py index 9d92e7b3..2b0e98c6 100644 --- a/datafaker/dialects.py +++ b/datafaker/dialects.py @@ -3,6 +3,7 @@ from collections.abc import Mapping from typing import Any, Optional, TypeVar +import snowflake.sqlalchemy.custom_types as snowflake_types from sqlalchemy import Column, Select, Table from sqlalchemy.ext.compiler import compiles from sqlalchemy.schema import CreateSchema, CreateTable @@ -515,3 +516,19 @@ def compile_random_mssql(_element: Random, _compiler: Any, **_kw: Any) -> str: same random number gets chosen for each row. """ return "NEWID()" + + +@compiles(snowflake_types.TIMESTAMP_LTZ) +def compile_timestampltz_generic( + _element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any +) -> str: + """Render TIMESTAMP_LTZ as DateTime.""" + return compiler.process(DateTime(timezone=False), **kw) + + +@compiles(snowflake_types.TIMESTAMP_LTZ, "snowflake") +def compile_timestampltz_ddb( + element: snowflake_types.TIMESTAMP_LTZ, compiler: Any, **kw: Any +) -> str: + """Render TIMESTAMP_LTZ as TIMESTAMP_LTZ.""" + return compiler.process(element, **kw) diff --git a/datafaker/interactive/generators.py b/datafaker/interactive/generators.py index 5d6dd0a8..cde6d532 100644 --- a/datafaker/interactive/generators.py +++ b/datafaker/interactive/generators.py @@ -8,7 +8,7 @@ from enum import Enum from typing import Any, Callable, Optional, cast -from sqlalchemy import Column, and_, literal_column, select +from sqlalchemy import Column, and_, select from sqlalchemy.types import Integer from datafaker.db_utils import MaybeAsyncEngine, primary_private_fks, table_is_private @@ -38,6 +38,7 @@ schema_qualified_name, set_property, split_column_full_name, + text_list, ) @@ -161,7 +162,8 @@ def do(self, column_roles: RoleEntry | None) -> RoleEntry | None: """Perform the command.""" if column_roles is None: return None - column_roles.new.remove(self.role) + if self.role in column_roles.new: + column_roles.new.remove(self.role) return column_roles @@ -198,6 +200,9 @@ class GeneratorCmd(DbCmd): PRIMARY_PRIVATE_TEXT = "Primary Private" SECONDARY_PRIVATE_TEXT = "Secondary Private on columns {0}" NOT_PRIVATE_TEXT = "Not private" + REQUIRES_NO_SOURCE_DATA_TEXT = ( + "{0}. {2}{1}{3} requires no data from the source database." + ) REQUIRES_SOURCE_DATA_TEXT = ( "{0}. {2}{1}{3} requires the following data from the source database:" ) @@ -472,7 +477,7 @@ def _copy_roles(self) -> None: """Set role information in the configuration.""" for table_name, table_roles in self.roles.items(): for column_name, entry in table_roles.items(): - if entry.new: + if entry.new or entry.old: set_property( self.config, ["tables", table_name, "columns", column_name, "roles"], @@ -534,8 +539,8 @@ def _print_role_changes(self) -> int: "Changing role set of column {0} of table {1} from {2} to {3}", column_name, table_name, - ", ".join(str(oe) for oe in entry.old), - ", ".join(str(ne) for ne in entry.new), + text_list((e.value for e in entry.old), "empty"), + text_list((e.value for e in entry.new), "empty"), ) return count @@ -904,7 +909,7 @@ def _print_values_queried(self, table_name: str, n: int, prop: Proposer) -> None theme = get_active_theme() if not prop.select_aggregate_clauses() and not prop.custom_queries(): self.print( - "{0}. {2}{1}{3} requires no data from the source database.", + self.REQUIRES_NO_SOURCE_DATA_TEXT, n, prop.name(), theme.function, @@ -1027,12 +1032,13 @@ def _print_select_aggregate_query(self, table_name: str, prop: Proposer) -> None def _get_column_data( self, count: int, to_str: Callable[[Any], str] = repr ) -> list[list[str]]: - columns = self._get_column_names() - col_exprs = [literal_column(col) for col in columns] - nonnull_clauses = [literal_column(col).isnot(None) for col in columns] + column_names = self._get_column_names() + table = self.table_metadata() + columns = [table.columns[c] for c in column_names] + nonnull_clauses = [col.isnot(None) for col in columns] stmt = ( - select(*col_exprs) - .select_from(self.table_metadata()) + select(*columns) + .select_from(table) .where(and_(*nonnull_clauses)) .order_by(Random()) .limit(count) diff --git a/datafaker/interactive/missingness.py b/datafaker/interactive/missingness.py index 60940a36..bc41557c 100644 --- a/datafaker/interactive/missingness.py +++ b/datafaker/interactive/missingness.py @@ -215,7 +215,7 @@ def _copy_entries(self) -> None: "kwargs": { "patterns": f'SRC_STATS["{src_stat_key}"]["results"]' }, - "columns": [c.name for c in entry.new_type.columns], + "columns": [str(c.name) for c in entry.new_type.columns], } ] src_stats.append( diff --git a/datafaker/interactive/table.py b/datafaker/interactive/table.py index 7a2852c0..107a53db 100644 --- a/datafaker/interactive/table.py +++ b/datafaker/interactive/table.py @@ -3,7 +3,7 @@ from dataclasses import dataclass from typing import Any, cast -from sqlalchemy import func, literal_column, select +from sqlalchemy import func, select from datafaker.dialects import Random from datafaker.interactive.base import ( @@ -469,26 +469,21 @@ def complete_data( return [] return self.get_column_completions(text) - def print_column_data(self, column: str, count: int, min_length: int) -> None: + def print_column_data(self, column_name: str, count: int, min_length: int) -> None: """ Print a sample of data from a certain column of the current table. - :param column: The name of the column to report on. + :param column_name: The name of the column to report on. :param count: The number of rows to sample. :param min_length: The minimum length of text to choose from (0 for any text). """ - col_expr = literal_column(column) + table = self.table_metadata() + column = table.columns[column_name] if 0 < min_length: - where_clause = func.length(col_expr) >= min_length + where_clause = func.length(column) >= min_length else: - where_clause = col_expr.isnot(None) - stmt = ( - select(col_expr) - .select_from(self.table_metadata()) - .where(where_clause) - .order_by(Random()) - .limit(count) - ) + where_clause = column.isnot(None) + stmt = select(column).where(where_clause).order_by(Random()).limit(count) with self.sync_engine.connect() as connection: result = connection.execute(stmt) self.columnize([str(x[0]) for x in result.all()]) diff --git a/datafaker/main.py b/datafaker/main.py index b180b97b..096ae305 100644 --- a/datafaker/main.py +++ b/datafaker/main.py @@ -177,6 +177,10 @@ def make_tables( file_okay=False, dir_okay=True, ), + views_to_tables: bool = Option( + False, + help=("Generate all views as tables in the synthetic version."), + ), ) -> None: """Make a YAML file representing the tables in the schema. @@ -192,7 +196,8 @@ def make_tables( content = make_tables_file( get_source_dsn(), get_source_schema(), - parquet_dir, + views_to_tables=views_to_tables, + parquet_dir=parquet_dir, ) orm_file_path.write_text(content, encoding="utf-8") logger.debug("%s created.", orm_file) diff --git a/datafaker/make.py b/datafaker/make.py index d9ddf25b..1fdb10ef 100644 --- a/datafaker/make.py +++ b/datafaker/make.py @@ -10,6 +10,7 @@ from typing import Any, Final, Optional, Tuple, Type import pandas as pd +import snowflake.sqlalchemy.custom_types as snowflake_types import snsql import typer import yaml @@ -58,6 +59,8 @@ if issubclass(entry, BaseProvider) and entry.__module__ == "datafaker.providers": PROVIDER_IMPORTS.append(entry_name) +DEFAULT_UNIQUE_CONSTRAINT_TRIES: int | None = 500 + @dataclass class VocabularyTableGeneratorInfo: @@ -378,6 +381,10 @@ def get_result_mappings( generator=_string_generator, choice=True, ), + snowflake_types.TIMESTAMP_LTZ: GeneratorInfo( + generator="generic.datetime.datetime", + arg_types={"start": int, "end": int}, + ), } @@ -654,6 +661,8 @@ def get_generation_info( max_unique_constraint_tries = get_property_or_none( config, "max-unique-constraint-tries", int ) + if max_unique_constraint_tries is None: + max_unique_constraint_tries = DEFAULT_UNIQUE_CONSTRAINT_TRIES return GenerationInfo( provider_imports=PROVIDER_IMPORTS, row_generator_module_name=row_generator_module_name, @@ -697,6 +706,7 @@ def _generate_vocabulary_table( def make_tables_file( db_dsn: str, schema_name: Optional[str], + views_to_tables: bool, parquet_dir: Optional[Path] = None, engine: Optional[Engine] = None, ) -> str: @@ -704,7 +714,9 @@ def make_tables_file( if engine is None: engine = get_sync_engine(create_db_engine(db_dsn, schema_name=schema_name)) - metadata = get_metadata(engine, schema_name=schema_name) + metadata = get_metadata( + engine, schema_name=schema_name, views_to_tables=views_to_tables + ) meta_dict = metadata_to_dict(metadata, schema_name, engine, parquet_dir) if parquet_dir is not None: diff --git a/datafaker/proposers/__init__.py b/datafaker/proposers/__init__.py index bee4c536..98c969a6 100644 --- a/datafaker/proposers/__init__.py +++ b/datafaker/proposers/__init__.py @@ -17,7 +17,7 @@ MultivariateNormalProposerFactory, ) from datafaker.proposers.extract import DateComponentExtractProposerFactory -from datafaker.proposers.intervals import DateAfterProposerFactory +from datafaker.proposers.intervals import CopyProposerFactory, DateAfterProposerFactory from datafaker.proposers.mimesis import ( MimesisDateProposerFactory, MimesisDateTimeProposerFactory, @@ -57,6 +57,7 @@ def everything_factory(config: Mapping, metadata: MetaData) -> ProposerFactory: NullPartitionedNormalProposerFactory(config, metadata), NullPartitionedLogNormalProposerFactory(config, metadata), DateAfterProposerFactory(config, metadata), + CopyProposerFactory(config, metadata), DateComponentExtractProposerFactory(config, metadata), IncrementProposerFactory(), ) diff --git a/datafaker/proposers/base.py b/datafaker/proposers/base.py index 9e677326..6b456ca0 100644 --- a/datafaker/proposers/base.py +++ b/datafaker/proposers/base.py @@ -2,19 +2,21 @@ import re from abc import ABC, abstractmethod -from collections.abc import Mapping, Sequence +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass from typing import Any, Union import mimesis import mimesis.locales from sqlalchemy import Column, Engine, Join, Table, func, select from sqlalchemy.exc import DatabaseError +from sqlalchemy.orm import aliased from sqlalchemy.types import Integer, Numeric, String, TypeEngine from typing_extensions import Self from datafaker.dialects import StdDev from datafaker.providers import DistributionProvider -from datafaker.utils import logger +from datafaker.utils import T, logger NumericType = Union[int, float] @@ -306,6 +308,93 @@ def fit_from_buckets(xs: Sequence[NumericType], ys: Sequence[NumericType]) -> fl return sum_diff_squared / (count * count) +class ForeignKeyRelationship: + """How a table is related.""" + + def __init__(self, fk_column: Column, target_column: Column) -> None: + """ + Initialise a ``ForeignKeyRelationship``. + + :param fk_column: The column containing the foreign key (the source). + :param target_column: The column that ``fk_column`` matches in + the target table. + """ + self.fk_column = fk_column + table = aliased(target_column.table) + self.target_column = table.columns[target_column.name] + + def __repr__(self): + """Render the relationship as a JOIN.""" + return ( + f"" + ) + + def target_table(self) -> Table: + """Get the related table.""" + return self.target_column.table + + def target_table_name(self) -> str: + """ + Get the name of the target table. + + Don't use ``target_table().name`` because this will give the + name of the aliased table. + """ + return str(self.target_column.table.original.name) + + +def _get_one(seq: Iterable[T]) -> T | None: + """Get an arbitrary member of the sequence, or None if it is empty.""" + for elt in seq: + return elt + return None + + +def make_foreign_key_relationship(fk_column: Column) -> ForeignKeyRelationship | None: + """Make a ``ForeignKeyRelationship`` from a foreign key column.""" + if (elt := _get_one(fk_column.foreign_keys)) is None: + return None + return ForeignKeyRelationship(fk_column, elt.column) + + +@dataclass +class RelatedColumn: + """A column in a related table, and how it is related.""" + + target: Column + relationship: ForeignKeyRelationship | None + + def add_from_to_query(self, query: Any) -> Any: + """Take a query and add a ``select_from`` and maybe ``join`` as appropriate.""" + if self.relationship is None: + return query.select_from(self.target.table) + query = query.select_from(self.relationship.fk_column.table) + return self.add_join_to_query(query) + + def add_join_to_query(self, query: Any) -> Any: + """Take a query and maybe add ``join`` as appropriate.""" + rel = self.relationship + if rel is None: + return query + return query.join( + rel.target_column.table, + onclause=rel.fk_column == rel.target_column, + ) + + def target_table(self) -> Table | None: + """Get the related table, or None if it's the same table.""" + if self.relationship is None: + return None + return self.relationship.target_table() + + def target_table_name(self) -> str | None: + """Get the name of the target table, or None if it's the same table.""" + if self.relationship is None: + return None + return self.relationship.target_table_name() + + class Buckets: """ Measured buckets for a real distribution. @@ -323,12 +412,12 @@ def __init__( mean: float, stddev: float, count: int, - join_tables: list[Table] | None = None, + anchor_relation: RelatedColumn | None = None, ): """Initialise a Buckets object.""" self.mean = mean self.stddev = stddev - query = self._get_bucket_query(table, column, join_tables) + query = self._get_bucket_query(table, column, anchor_relation) with engine.connect() as connection: raw_buckets = connection.execute(query.group_by("b")) self.buckets: Sequence[int] = [0] * 10 @@ -350,7 +439,7 @@ def _get_bucket_query( self, table: Table | Join, column: Any, - join_tables: list[Table] | None, + anchor_relation: RelatedColumn | None = None, ) -> Any: """Make the query for half-sd buckets.""" bottom = self.mean - 2 * self.stddev @@ -359,8 +448,8 @@ def _get_bucket_query( func.count(column).label("f"), # pylint: disable=not-callable ((func.floor(column) - bottom) / width).label("b"), ).select_from(table) - for jt in join_tables or []: - query = query.join(jt) + if anchor_relation is not None: + query = anchor_relation.add_join_to_query(query) return query @classmethod @@ -369,7 +458,7 @@ def make_buckets( engine: Engine, table: Table | Join, column: Any, - join_tables: list[Table] | None = None, + anchor_relation: RelatedColumn | None = None, ) -> Self | None: """ Construct a Buckets object. @@ -383,14 +472,16 @@ def make_buckets( :param engine: SQLAlchemy engine. :param table: SQLAlchemy table (or joined tables) to pull data from. :param column: SQLAlchemy column or expression to measure. + :param anchor_relation: A relationship to another column whose table + needs to be joined. """ query = select( func.avg(column).label("mean"), StdDev(column).label("stddev"), func.count(column).label("count"), # pylint: disable=not-callable ).select_from(table) - for jt in join_tables or []: - query = query.join(jt) + if anchor_relation is not None: + query = anchor_relation.add_join_to_query(query) with engine.connect() as connection: result = connection.execute(query).first() if result is None or result.stddev is None or getattr(result, "count") < 2: @@ -403,7 +494,7 @@ def make_buckets( result.mean, result.stddev, getattr(result, "count"), - join_tables, + anchor_relation, ) except DatabaseError as exc: logger.debug("Failed to instantiate Buckets object: %s", exc) diff --git a/datafaker/proposers/choice.py b/datafaker/proposers/choice.py index acf0e1f4..38e475f2 100644 --- a/datafaker/proposers/choice.py +++ b/datafaker/proposers/choice.py @@ -6,17 +6,7 @@ from abc import abstractmethod from typing import Any, Sequence, Union -from sqlalchemy import ( - Column, - CursorResult, - Engine, - desc, - func, - literal_column, - select, - table, - text, -) +from sqlalchemy import Column, CursorResult, Engine, desc, func, select from datafaker.dialects import Random from datafaker.proposers.base import ( @@ -25,7 +15,6 @@ dist_gen, fit_from_buckets, ) -from datafaker.utils import schema_qualified_name NumericType = Union[int, float] @@ -57,12 +46,10 @@ def zipf_distribution(total: int, bins: int) -> typing.Generator[int, None, None def _choice_stmt( # pylint: disable=R0913,R0917 - column_name: str, - table_name: str, + column: Column, store_counts: bool, sample_count: int | None, suppress_count: int, - table_sql: str | None = None, ) -> Any: """Build a SQLAlchemy SELECT for gathering choice value distributions. @@ -70,13 +57,10 @@ def _choice_stmt( # pylint: disable=R0913,R0917 TOP/newid() on MS-SQL. MS-SQL also forbids ORDER BY inside a subquery without TOP; this function never emits such a clause. """ - col = literal_column(f'"{column_name}"') - tbl = text(table_sql) if table_sql else table(table_name) if sample_count is not None: sample_sub = ( - select(col.label("value")) - .where(col.isnot(None)) - .select_from(tbl) + select(column.label("value")) + .where(column.isnot(None)) .order_by(Random()) .limit(sample_count) .subquery("_inner") @@ -92,12 +76,11 @@ def _choice_stmt( # pylint: disable=R0913,R0917 else: counted_sub = ( select( - col.label("value"), - func.count(col).label("count"), # pylint: disable=E1102 + column.label("value"), + func.count(column).label("count"), # pylint: disable=E1102 ) - .where(col.isnot(None)) - .select_from(tbl) - .group_by(col) + .where(column.isnot(None)) + .group_by(column) .subquery("_counted") ) out_cols = [counted_sub.c.value] @@ -119,31 +102,26 @@ class ChoiceProposer(Proposer): # pylint: disable=too-many-arguments too-many-positional-arguments def __init__( self, - table_name: str, - column_name: str, + column: Column, values: list[Any], counts: list[int], sample_count: int | None = None, suppress_count: int = 0, dialect: Any = None, - table_sql: str | None = None, ) -> None: """Initialise a ChoiceProposer.""" super().__init__() - self.table_name = table_name - self.column_name = column_name + self.column = column self.values = values estimated_counts = self.get_estimated_counts(counts) self._fit = fit_from_buckets(counts, estimated_counts) extra_comment = " and their counts" if self.STORE_COUNTS else "" stmt = _choice_stmt( - column_name, - table_name, + column, self.STORE_COUNTS, sample_count, suppress_count, - table_sql=table_sql, ) compile_opts: dict[str, Any] = {"compile_kwargs": {"literal_binds": True}} if dialect is not None: @@ -153,28 +131,28 @@ def __init__( if suppress_count == 0: if sample_count is None: self._comment = ( - f"All the values{extra_comment} that appear in column {column_name}" - f" of table {table_name}" + f"All the values{extra_comment} that appear in column {column.name}" + f" of table {column.table.name}" ) self._annotation = None else: self._comment = ( - f"The values{extra_comment} that appear in column {column_name}" - f" of a random sample of {sample_count} rows of table {table_name}" + f"The values{extra_comment} that appear in column {column.name}" + f" of a random sample of {sample_count} rows of table {column.table.name}" ) self._annotation = "sampled" else: if sample_count is None: self._comment = ( - f"All the values{extra_comment} that appear in column {column_name}" - f" of table {table_name} more than {suppress_count} times" + f"All the values{extra_comment} that appear in column {column.name}" + f" of table {column.table.name} more than {suppress_count} times" ) self._annotation = "suppressed" else: self._comment = ( f"The values{extra_comment} that appear more than {suppress_count} times" - f" in column {column_name}, out of a random sample of {sample_count} rows" - f" of table {table_name}" + f" in column {column.name}, out of a random sample of {sample_count} rows" + f" of table {column.table.name}" ) self._annotation = "sampled and suppressed" @@ -185,7 +163,7 @@ def get_estimated_counts(self, counts: list[int]) -> list[int]: def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" return { - "a": f'SRC_STATS["auto__{self.table_name}__{self.column_name}"]["results"]', + "a": f'SRC_STATS["auto__{self.column.table.name}__{self.column.name}"]["results"]', } def name(self) -> str: @@ -206,7 +184,7 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: qs = super().custom_queries() return { **qs, - f"auto__{self.table_name}__{self.column_name}": { + f"auto__{self.column.table.name}__{self.column.name}": { "query": self._query, "comments": [self._comment], }, @@ -350,23 +328,19 @@ def get_proposers( # pylint: disable=too-many-locals if len(columns) != 1: return [] column = columns[0] - column_name = column.name - table_name = column.table.name - table_sql = schema_qualified_name(table_name, engine) src_table = column.table dialect = engine.dialect - col = literal_column(f'"{column_name}"') src_table = column.table # preserves schema for schema-qualified databases generators = [] with engine.connect() as connection: stmt_count = ( select( - col.label("v"), - func.count(col).label("f"), # pylint: disable=E1102 + column.label("v"), + func.count(column).label("f"), # pylint: disable=E1102 ) .select_from(src_table) - .group_by(col) - .order_by(desc(func.count(col))) # pylint: disable=E1102 + .group_by(column) + .order_by(desc(func.count(column))) # pylint: disable=E1102 .limit(MAXIMUM_CHOICES + 1) ) results = connection.execute(stmt_count) @@ -375,62 +349,50 @@ def get_proposers( # pylint: disable=too-many-locals if vg.counts: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name, - column_name, + column, vg.cvs, vg.counts, dialect=dialect, - table_sql=table_sql, ), ] if vg.counts_not_suppressed: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name=table_name, - column_name=column_name, + column=column, values=vg.cvs_not_suppressed, counts=vg.counts_not_suppressed, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), ] inner = ( - select(col.label("v")) + select(column.label("v")) .select_from(src_table) .order_by(Random()) .limit(self.SAMPLE_COUNT) @@ -450,64 +412,52 @@ def get_proposers( # pylint: disable=too-many-locals if vg.counts: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name, - column_name, + column, vg.cvs, vg.counts, sample_count=self.SAMPLE_COUNT, dialect=dialect, - table_sql=table_sql, ), ] if vg.counts_not_suppressed: generators += [ ZipfChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), UniformChoiceProposer( - table_name, - column_name, + column, vg.values_not_suppressed, vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), WeightedChoiceProposer( - table_name=table_name, - column_name=column_name, + column=column, values=vg.cvs_not_suppressed, counts=vg.counts_not_suppressed, sample_count=self.SAMPLE_COUNT, suppress_count=self.SUPPRESS_COUNT, dialect=dialect, - table_sql=table_sql, ), ] return generators diff --git a/datafaker/proposers/intervals.py b/datafaker/proposers/intervals.py index e6bb11fc..c6142cba 100644 --- a/datafaker/proposers/intervals.py +++ b/datafaker/proposers/intervals.py @@ -3,23 +3,28 @@ from collections.abc import Mapping, Sequence from typing import Any -from sqlalchemy import Column, Engine, ForeignKey, MetaData, func, select +from sqlalchemy import Column, Engine, MetaData, func, select from sqlalchemy.types import Date, DateTime -from datafaker.db_utils import get_fk_column_between from datafaker.dialects import Random, SecondsDifference, StdDev -from datafaker.proposers.base import Buckets, Proposer, ProposerFactory, get_column_type +from datafaker.proposers.base import ( + Buckets, + ForeignKeyRelationship, + Proposer, + ProposerFactory, + RelatedColumn, + get_column_type, + make_foreign_key_relationship, +) from datafaker.providers import AnchoredProvider from datafaker.utils import get_property, procrustes -RelatedColumn = tuple[ForeignKey | None, Column] - def _set_roles_for_column( out: dict[str, list[RelatedColumn]], - fk: ForeignKey | None, + fk: ForeignKeyRelationship | None, column: Column, - column_config: Mapping, + columns_config: Mapping, ) -> None: """ Set new entries in ``out`` based on the roles ``column`` has. @@ -29,22 +34,22 @@ def _set_roles_for_column( that role and ``fk`` is the foreign key in the table ``column`` appears in that points to the table ``fcol`` appears in, or None if ``column`` and ``fcol`` are in the same table. - :param fk: Foreign key to the table ``column`` appears in (appears in - new entries set in ``out``). + :param fk: Foreign key relationship to the table ``column`` appears in + (appears in new entries set in ``out``). :param column: The column to be checked for roles. - :param column_config: The ``tables: : columns:`` section + :param columns_config: The ``tables: : columns:`` section of the ``config.yaml`` file. """ - roles: list[Any] = get_property(column_config, [column.name, "roles"], []) + roles: list[Any] = get_property(columns_config, [column.name, "roles"], []) for role in roles: - pair: RelatedColumn = (fk, column) + rc = RelatedColumn(column, fk) if role not in out: - out[role] = [pair] + out[role] = [rc] else: - out[role].append(pair) + out[role].append(rc) -def _get_roles( +def _get_all_relative_roles( config: Mapping, column: Column, ) -> dict[str, list[RelatedColumn]]: @@ -60,24 +65,95 @@ def _get_roles( """ table = column.table tables_config: dict[str, Any] = get_property(config, "tables", {}) - table_conf: dict[str, Any] = get_property( + columns_config: dict[str, Any] = get_property( tables_config, [str(table.name), "columns"], {} ) role_to_fk_columns: dict[str, list[RelatedColumn]] = {} for col in table.columns: - _set_roles_for_column(role_to_fk_columns, None, col, table_conf) + _set_roles_for_column(role_to_fk_columns, None, col, columns_config) # look for roles in related tables - if col.foreign_keys: - fk = list(col.foreign_keys)[0] - target_table = fk.column.table + if (fk_relationship := make_foreign_key_relationship(col)) is not None: ft_conf: dict[str, Any] = get_property( - tables_config, [str(target_table.name), "columns"], {} + tables_config, [fk_relationship.target_table_name(), "columns"], {} ) - for fcol in target_table.columns: - _set_roles_for_column(role_to_fk_columns, fk, fcol, ft_conf) + for fcol in fk_relationship.target_table().columns: + _set_roles_for_column( + role_to_fk_columns, fk_relationship, fcol, ft_conf + ) return role_to_fk_columns +def _not_anchored_on_ourselves(column: Column, fk_col: RelatedColumn) -> bool: + """ + Return whether these columns are the exact same thing. + + :param column: The column we are wanting to generate for. + :param fk_col: The column we are considering using as an anchor. + :return: False if ``fk_col`` targets the same column as ``column`` + not through a foreign key, so True if ``fk_col`` is through a FK + or if it doesn't target the same column. + """ + return not (fk_col.relationship is None and fk_col.target == column) + + +def _not_anchoring_nonnullable_to_nullable( + column: Column, fk_col: RelatedColumn +) -> bool: + """ + Return True if ``column`` is nullable or ``fk_col`` is not. + + We are counting ``fk_col`` as nullabe if either the target is nullable + or the foreign key itself. + + This is required if ``fk_col`` can be copied to ``col``. + """ + fk_nullable = ( + fk_col.relationship is not None + and fk_col.relationship.fk_column.nullable is not False + ) + fk_col_nullable = fk_nullable or fk_col.target.nullable + return column.nullable or not fk_col_nullable + + +def _not_copying_incompatible_fk(column: Column, fk_col: RelatedColumn) -> bool: + """Return True if ``column`` and ``fk_col`` are incompatible foreign key types.""" + target = fk_col.target + if not column.foreign_keys: + return not target.foreign_keys + if not target.foreign_keys: + return False + tfks = {c.target_fullname for c in target.foreign_keys} + cfks = {c.target_fullname for c in column.foreign_keys} + return tfks == cfks + + +def _get_roles( + config: Mapping, + column: Column, + role_name: str, +) -> list[RelatedColumn]: + """ + Get all the ``role_name`` roles in this table or a related table. + + :param role_name: The role to return. + :return: The list of all columns in the same table as ``column`` or in + tables directly related to that table that have the named role. This + list has the following columns filtered out: Nullable columns (if + ``column`` is nonnullable), foreign keys if ``column`` is not a + foreign key, non-foreign keys if ``column`` is a foreign key + or foreign keys if ``column`` is a foreign key to a different table. + """ + roles = _get_all_relative_roles(config, column) + if role_name not in roles: + return [] + return [ + fk_col + for fk_col in roles[role_name] + if _not_anchored_on_ourselves(column, fk_col) + and _not_anchoring_nonnullable_to_nullable(column, fk_col) + ] + + def coerce_to_datetime( d: datetime.date | datetime.datetime, tzdonor: datetime.datetime, @@ -103,7 +179,7 @@ def __init__( sd: float, mean: float, column: Column, - anchor: Column, + anchor_related: RelatedColumn, engine: Engine, buckets: Buckets | None = None, ): @@ -116,13 +192,13 @@ def __init__( :param column: The column to generate. :param anchor: The anchor column, which must be in the same table as column or a related table. - :param dialect: The dialect of the source database. + :param engine: The source database. :param buckets: The buckets for the intervals in the source database. """ super().__init__() self._sd = sd self._mean = mean - self._anchor = anchor + self._anchor_related = anchor_related self._column = column self._engine = engine self._provider = AnchoredProvider(metadata=metadata) @@ -134,27 +210,44 @@ def __init__( [(b - coerce_to_datetime(a, b)).total_seconds() for (a, b) in intervals] ) + @classmethod + def get_query( + cls, + column: Column, + anchor_related: RelatedColumn, + ) -> Any: + """Create the query for the summary of interval lengths.""" + anchor = anchor_related.target + return anchor_related.add_from_to_query( + select( + func.avg(SecondsDifference(column, anchor)).label("mean"), + StdDev(SecondsDifference(column, anchor)).label("sd"), + ) + ) + def function_name(self) -> str: """Get the name of the generator function to call.""" - if self._column.table == self._anchor.table: + if self._anchor_related.relationship is None: return "generic.anchored_provider.normal_date" return "generic.anchored_provider.normal_date_fk" def name(self) -> str: """Get the name of the generator.""" fname = self.function_name() - aname = self._anchor.name - atable = self._anchor.table - if atable == self._column.table: + aname = self._anchor_related.target.name + rel = self._anchor_related.relationship + if rel is None: return f"{fname} [anchored to {aname}]" - return f"{fname} [anchored to {aname} of table {atable.name}]" + atable = rel.target_table_name() + fk_name = rel.fk_column.name + return f"{fname} [anchored to {aname} of table {atable} (via {fk_name})]" def nominal_kwargs(self) -> dict[str, Any]: """Get the arguments to be entered into ``config.yaml``.""" column = self._column - anchor = self._anchor - (fk_col, fk) = get_fk_column_between(column.table, anchor.table) - if fk_col is None or fk is None: + anchor = self._anchor_related.target + rel = self._anchor_related.relationship + if rel is None: return { "mean_seconds": ( f'SRC_STATS["auto__{column.table.name}"]' @@ -167,33 +260,34 @@ def nominal_kwargs(self) -> dict[str, Any]: "anchor": f'GENERATED_ROW["{anchor.name}"]', } key = f"auto__interval__{column.table.name}__{column.name}" + fk_col = rel.fk_column.name + on_col = rel.target_column.name return { "dst_db_conn": "dst_db_conn", "anchor_column": f'"{anchor.name}"', - "table": f'"{anchor.table.name}"', + "table": f'"{rel.target_table_name()}"', "mean_seconds": (f'SRC_STATS["{key}"]["results"][0]["mean"]'), "sd_seconds": (f'SRC_STATS["{key}"]["results"][0]["sd"]'), - "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', - "on_column": f'"{fk.column.name}"', + "anchor_row": f'GENERATED_ROW["{fk_col}"]', + "on_column": f'"{on_col}"', } def actual_kwargs(self) -> dict[str, Any]: """Get the kwargs (summary statistics) this generator was instantiated with.""" - column = self._column - anchor = self._anchor - (_, fk) = get_fk_column_between(column.table, anchor.table) - if fk is None: + if self._anchor_related.relationship is None: return { "mean_seconds": self._sd, "sd_seconds": self._mean, "anchor": "1970-01-01", } + anchor = self._anchor_related.target + on_col = self._anchor_related.relationship.target_column.name return { "anchor_column": anchor.name, "table": anchor.table.name, "mean_seconds": self._sd, "sd_seconds": self._mean, - "on_column": fk.column.name, + "on_column": on_col, } def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: @@ -202,10 +296,10 @@ def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: This is for anchors in the same table. """ - column = self._column - anchor = self._anchor - if column.table != anchor.table: + if self._anchor_related.relationship is not None: return {} + column = self._column + anchor = self._anchor_related.target mean_q = func.avg(SecondsDifference(column, anchor)) sd_q = StdDev(SecondsDifference(column, anchor)) @@ -243,23 +337,17 @@ def custom_queries(self) -> dict[str, dict[str, Any]]: This is for anchors in a related table. """ column = self._column - anchor = self._anchor - if column.table == anchor.table: + rel = self._anchor_related + if rel.relationship is None: return {} - query = ( - select( - func.avg(SecondsDifference(column, anchor)).label("mean"), - StdDev(SecondsDifference(column, anchor)).label("sd"), - ) - .select_from(column.table) - .join(anchor.table) - ) + query = DateAfterProposer.get_query(column, rel) return { f"auto__interval__{column.table.name}__{column.name}": { "comments": [ "Mean and standard deviation of the length of time between" - f" column {anchor.name} of table {anchor.table.name}" - f" and column {column.name} of table {column.table.name}." + f" column {rel.target.name} of table {rel.target.table.name}" + f" and column {column.name} of table {column.table.name}" + f" (joined through {rel.relationship.fk_column.name})" ], "query": str( query.compile( @@ -288,7 +376,8 @@ def generate_intervals( :param count: The number of intervals to generate. :return: Each pair consists of an anchor and the generated datetime. """ - query = select(self._anchor).select_from(self._anchor.table) + anchor = self._anchor_related.target + query = select(anchor).select_from(anchor.table) query = query.order_by(Random()).limit(count) with self._engine.connect() as conn: anchors = conn.execute(query).scalars().all() @@ -312,18 +401,14 @@ def __init__(self, config: Mapping, metadata: MetaData): self._config = config self._metadata = metadata - def make_date_after_proposers( - self, engine: Engine, column: Column, anchor: Column + def _make_date_after_proposers( + self, + engine: Engine, + column: Column, + anchor_related: RelatedColumn, ) -> list[DateAfterProposer]: """Create a ``DateAfterProposer`` object.""" - query = select( - func.avg(SecondsDifference(column, anchor)).label("mean"), - StdDev(SecondsDifference(column, anchor)).label("sd"), - ).select_from(column.table) - join_tables = [] - if anchor.table != column.table: - join_tables = [anchor.table] - query = query.join(anchor.table) + query = DateAfterProposer.get_query(column, anchor_related) with engine.connect() as connection: result = connection.execute(query).first() if result is None or result.sd is None: @@ -331,8 +416,8 @@ def make_date_after_proposers( buckets = Buckets.make_buckets( engine, column.table, - SecondsDifference(column, anchor), - join_tables, + SecondsDifference(column, anchor_related.target), + anchor_related, ) return [ DateAfterProposer( @@ -340,7 +425,7 @@ def make_date_after_proposers( result.sd, result.mean, column, - anchor, + anchor_related, engine=engine, buckets=buckets, ) @@ -358,14 +443,151 @@ def get_proposers( ct = get_column_type(column) if not isinstance(ct, (Date, DateTime)): return [] - roles = _get_roles(self._config, column) - if "start" not in roles: + other_start_columns = _get_roles(self._config, column, "start") + return [ + prop + for anchor in other_start_columns + for prop in self._make_date_after_proposers(engine, column, anchor) + ] + + +class CopyProposer(Proposer): + """Proposer that proposes copying another column.""" + + # pylint: disable=too-many-arguments too-many-positional-arguments + def __init__( + self, + metadata: MetaData, + column: Column, + anchor_related: RelatedColumn, + engine: Engine, + ): + """ + Initialise a date after proposer. + + :param metadata: The metadata of the source database. + :param column: The column to generate. + :param anchor: The anchor column, which must be in the same table as column + or a related table. + :param engine: The source database. + :param buckets: The buckets for the intervals in the source database. + """ + super().__init__() + self._anchor_related = anchor_related + self._column = column + self._engine = engine + self._provider = AnchoredProvider(metadata=metadata) + + def function_name(self) -> str: + """Get the name of the generator function to call.""" + if self._anchor_related.relationship is None: + return "generic.anchored_provider.copy" + return "generic.anchored_provider.copy_fk" + + def name(self) -> str: + """Get the name of the generator.""" + fname = self.function_name() + aname = self._anchor_related.target.name + rel = self._anchor_related.relationship + if rel is None: + return f"{fname} [from {aname}]" + atable = rel.target_table_name() + on_col = rel.fk_column.name + return f"{fname} [from {aname} of table {atable} (via {on_col})]" + + def nominal_kwargs(self) -> dict[str, Any]: + """Get the arguments to be entered into ``config.yaml``.""" + anchor = self._anchor_related.target + rel = self._anchor_related.relationship + if rel is None: + return { + "anchor": f'GENERATED_ROW["{anchor.name}"]', + } + fk_col = rel.fk_column + on_col = rel.target_column + return { + "dst_db_conn": "dst_db_conn", + "anchor_column": f'"{anchor.name}"', + "table": f'"{rel.target_table_name()}"', + "anchor_row": f'GENERATED_ROW["{fk_col.name}"]', + "on_column": f'"{on_col.name}"', + } + + def actual_kwargs(self) -> dict[str, Any]: + """Get the kwargs (summary statistics) this generator was instantiated with.""" + anchor = self._anchor_related.target + if self._anchor_related.relationship is None: + return { + "anchor": "1970-01-01", + } + on_col = self._anchor_related.relationship.target_column + return { + "anchor_column": anchor.name, + "table": anchor.table.name, + "on_column": on_col.name, + } + + def select_aggregate_clauses(self) -> dict[str, dict[str, str]]: + """No source data required.""" + return {} + + def custom_queries(self) -> dict[str, dict[str, Any]]: + """No source data required.""" + return {} + + def fit(self, default: float = -1) -> float: + """Get this generator's fit against the real data.""" + return default + + def generate_data(self, count: int) -> list[Any]: + """Generate ``count`` random data points for this column.""" + anchor = self._anchor_related.target + query = select(anchor).select_from(anchor.table) + query = query.order_by(Random()).limit(count) + with self._engine.connect() as conn: + return list(conn.execute(query).scalars().all()) + + +class CopyProposerFactory(ProposerFactory): + """Proposer that proposes copying another column.""" + + def __init__(self, config: Mapping, metadata: MetaData): + """Initialize ``CopyProposerFactory``.""" + super().__init__() + self._config = config + self._metadata = metadata + + def make_copy_fk_proposers( + self, engine: Engine, column: Column, anchor: RelatedColumn + ) -> list[CopyProposer]: + """Create a ``CopyProposer`` object.""" + return [ + CopyProposer( + self._metadata, + column, + anchor, + engine=engine, + ) + ] + + def _can_copy_type(self, copy_from, copy_to) -> bool: + return type(copy_from) is type(copy_to) + + def get_proposers( + self, + columns: list[Column], + engine: Engine, + ) -> Sequence[Proposer]: + """Get all proposers of dates that might be anchored to another column.""" + if len(columns) != 1: return [] - other_start_columns = [ - fk_col for fk_col in roles["start"] if fk_col[1] != column + column = columns[0] + fk_cols = _get_roles(self._config, column, "source") + other_source_columns = [ + fk_col for fk_col in fk_cols if _not_copying_incompatible_fk(column, fk_col) ] return [ prop - for anchor in other_start_columns - for prop in self.make_date_after_proposers(engine, column, anchor[1]) + for anchor in other_source_columns + for prop in self.make_copy_fk_proposers(engine, column, anchor) ] diff --git a/datafaker/proposers/mimesis.py b/datafaker/proposers/mimesis.py index 451ad792..dbb9282f 100644 --- a/datafaker/proposers/mimesis.py +++ b/datafaker/proposers/mimesis.py @@ -5,7 +5,7 @@ import mimesis import mimesis.locales -from sqlalchemy import Column, Engine, cast, extract, func, literal_column, select +from sqlalchemy import Column, Engine, cast, extract, func, select from sqlalchemy.exc import SQLAlchemyError from sqlalchemy.types import Date, DateTime, Integer, Numeric, String, Time @@ -187,8 +187,7 @@ def make_singleton( cls, column: Column, engine: Engine, function_name: str ) -> Sequence[Proposer]: """Make the appropriate generation configuration for this column.""" - col_expr = literal_column(column.name) - year_expr = cast(extract("year", col_expr), Integer()) + year_expr = cast(extract("year", column), Integer()) min_expr = func.min(year_expr) # pylint: disable=E1111 max_expr = func.max(year_expr) # pylint: disable=E1111 stmt = select(min_expr.label("start"), max_expr.label("end")).select_from( diff --git a/datafaker/proposers/partitioned.py b/datafaker/proposers/partitioned.py index 595fcf81..6f8bc669 100644 --- a/datafaker/proposers/partitioned.py +++ b/datafaker/proposers/partitioned.py @@ -28,7 +28,7 @@ CovariateQuery, MultivariateNormalProposerFactory, ) -from datafaker.utils import T, get_property, logger +from datafaker.utils import T, get_property, logger, text_list NumericType = Union[int, float] @@ -37,23 +37,6 @@ MAXIMUM_CHOICES = 500 -def text_list(items: Iterable[str]) -> str: - """Concatenate the items with commas and one "and".""" - item_i = iter(items) - try: - last_item = next(item_i) - except StopIteration: - return "" - try: - so_far = next(item_i) - except StopIteration: - return last_item - for item in item_i: - so_far += ", " + last_item - last_item = item - return so_far + " and " + last_item - - @dataclass class RowPartition: """A partition where all the rows have the same pattern of NULLs.""" diff --git a/datafaker/providers.py b/datafaker/providers.py index 0263ef56..55faabf6 100644 --- a/datafaker/providers.py +++ b/datafaker/providers.py @@ -693,8 +693,8 @@ def normal_date_fk( :param sd_seconds: Standard deviation of the intervals' lengths in seconds. :param table: The name of the table for the anchor column. :param on_column: The name of the column in the foreign table - that ``row`` must match. - :param row: The value in the ``on_column`` column in table ``table`` + that ``anchor_row`` must match. + :param anchor_row: The value in the ``on_column`` column in table ``table`` for the row providing the start of the interval. :param anchor_column: The name of the column in ``table`` providing the start of the interval. @@ -711,3 +711,34 @@ def normal_date_fk( if not isinstance(out, (str, dt.date, dt.datetime)): return None return self.normal_date(mean_seconds, sd_seconds, out) + + def copy(self, anchor: Any) -> Any: + """Copy another column from the same table.""" + return anchor + + def copy_fk( + self, + dst_db_conn: Connection, + table: str, + on_column: str, + anchor_row: Any, + anchor_column: str, + ) -> Any: + """ + Copy a column from a different table. + + :param dst_db_conn: Connection to the destination database. + :param table: The name of the table for the anchor column. + :param on_column: The name of the column in the foreign table + that ``anchor_row`` must match. + :param anchor_row: The value in the ``on_column`` column in table ``table`` + for the row providing the source to copy. + :param anchor_column: The name of the column in ``table`` to copy. + :return: The value to be copied. + """ + mt = self._metadata.tables[table] + query = select(mt.c[anchor_column].label("out")).where( + mt.c[on_column] == anchor_row, + ) + anchor = dst_db_conn.execute(query).first() + return getattr(anchor, "out", None) diff --git a/datafaker/remove.py b/datafaker/remove.py index 0949e048..a6eae71c 100644 --- a/datafaker/remove.py +++ b/datafaker/remove.py @@ -68,5 +68,9 @@ def remove_db_tables(metadata: Optional[MetaData]) -> None: ) ) if metadata is None: - metadata = get_metadata(dst_engine, schema_name=schema_name) + metadata = get_metadata( + dst_engine, + schema_name=schema_name, + views_to_tables=False, + ) metadata.drop_all(dst_engine) diff --git a/datafaker/serialize_metadata.py b/datafaker/serialize_metadata.py index 35a066b7..fc2b555c 100644 --- a/datafaker/serialize_metadata.py +++ b/datafaker/serialize_metadata.py @@ -6,6 +6,7 @@ from typing import Any import parsy +import snowflake.sqlalchemy.custom_types as snowflake_types from sqlalchemy import Column, Dialect, Engine, ForeignKey, MetaData, Table from sqlalchemy.dialects import mssql, oracle, postgresql from sqlalchemy.sql import schema, sqltypes @@ -103,7 +104,7 @@ def time_type(type_: type, tz_type: type) -> ParserType: Make a parser for a SQL date/time type. Parses TYPE_NAME, TYPE_NAME(32), TYPE_NAME WITH TIME ZONE - or TYPE_NAME(32) WITH TIME ZONE + or TYPE_NAME(32) WITH TIME ZONE, also WITHOUT :param type_: The SQLAlchemy type we would like to parse. :param tz_type: The type to instantiate when precision or timezone is @@ -204,6 +205,15 @@ def _mssql_varbinary_parser() -> typing.Generator[ParserType, None, typing.Any]: string_type(sqltypes.VARCHAR), string_type(sqltypes.NVARCHAR), string_type(sqltypes.TEXT), + parsy.string("TIMESTAMP_NTZ").result(sqltypes.DATETIME), + parsy.string("TIMESTAMPNTZ").result(sqltypes.DATETIME), + parsy.string("TIMESTAMP_LTZ").result(snowflake_types.TIMESTAMP_LTZ), + parsy.string("TIMESTAMPLTZ").result(snowflake_types.TIMESTAMP_LTZ), + parsy.string("TIMESTAMP WITH LOCAL TIME ZONE").result( + snowflake_types.TIMESTAMP_LTZ + ), + parsy.string("TIMESTAMP_TZ").result(lambda: sqltypes.DATETIME(timezone=True)), + parsy.string("TIMESTAMPTZ").result(lambda: sqltypes.DATETIME(timezone=True)), time_type(sqltypes.TIMESTAMP, postgresql.types.TIMESTAMP), time_type(sqltypes.TIME, postgresql.types.TIME), ) diff --git a/datafaker/utils.py b/datafaker/utils.py index a6a15321..e7ce255e 100644 --- a/datafaker/utils.py +++ b/datafaker/utils.py @@ -457,6 +457,23 @@ def underline_error(e: SyntaxError) -> str: return "\n" + " " * start + "^" * (end - start) +def text_list(items: Iterable[str], empty="") -> str: + """Concatenate the items with commas and one "and".""" + item_i = iter(items) + try: + last_item = next(item_i) + except StopIteration: + return empty + try: + so_far = next(item_i) + except StopIteration: + return last_item + for item in item_i: + so_far += ", " + last_item + last_item = item + return so_far + " and " + last_item + + def gather_from_ast( errors: MutableSequence[tuple], name: str, diff --git a/docs/source/docker.rst b/docs/source/docker.rst index 62fcfe0a..9c21bb2b 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -23,7 +23,7 @@ and stores the files produced in a directory called ``output``: .. code-block:: shell mkdir output - docker run --rm --user $(id -u):$(id -g) --network host -e SRC_SCHEMA=public -e SRC_DSN=postgresql://tim:tim@localhost:5432/pagila -itv ./output:data datafaker + docker run --rm --user $(id -u):$(id -g) --network host -e SRC_SCHEMA=public -e SRC_DSN=postgresql://tim:tim@localhost:5432/pagila -itv ./output:/data datafaker You do need to create the output folder first. diff --git a/docs/source/faq.rst b/docs/source/faq.rst index a5700f02..82bc9dbb 100644 --- a/docs/source/faq.rst +++ b/docs/source/faq.rst @@ -1,19 +1,52 @@ FAQ === -Can datafaker work with two different schemas? +Can DataFaker work with two different schemas? ********************************************** -datafaker can only work with a single source schema and a single destination schema at a time. -However, you can choose for the destination schema to have a different name to the source schema by setting the ``DST_SCHEMA`` environment variable. +DataFaker can only work with a single source schema or a single destination schema at a time. +It is possible to use the same database for source and destination by choosing a different schema for source and destination, +for example by setting ``DST_SCHEMA=fake``. Which DBMSs does datafaker support? *********************************** -* datafaker most fully supports **PostgresSQL**, which it uses for its end-to-end functional tests. -* datafaker also supports **MariaDB**, as long as you don't set ``use-asyncio: true`` in your config. -* datafaker *might* work with **SQLite** but this is largely untested. -* datafaker may also work with SQL Server. - To connect to SQL Server, you will need to install `pyodbc `_ and an `ODBC driver `_, after which you should be able to use a DSN setting similar to ``SRC_DSN="mssql+pyodbc://username:password@hostname/dbname?driver=ODBC Driver 18 for SQL Server"``. +PostgreSQL +^^^^^^^^^^ -Please open a GitHub issue if you would like to see support for another DBMS. +Fully supported. + +Microsoft SQL Server +^^^^^^^^^^^^^^^^^^^^ + +Supported. Microsoft Transact SQL probably works as well. +You need to install an `ODBC driver `_. + +The DSN format is similar to: + +``mssql+pyodbc://:@127.0.0.1:1433/?driver=ODBC+Driver+18+for+SQL+Server&TrustServerCertificate=yes`` + +DuckDB +^^^^^^ + +Supported. +If you want to produce Parquet or CSV files as your output format, the easiest way is via a DuckDB database. +You do not need to install DuckDB to do this, simply use a destination DSN like ``duckdb:///./fake.db``, +then use ``datafaker dump-data --output my-csv-folder`` or ``datafaker dump-data --parquet --output my-parquet-folder``. + +An in-memory DuckDB database can be used as an empty database that you can augment with parquet files: + +``datafaker make-tables --parquet-dir my-parquet-input-folder`` + +Snowflake +^^^^^^^^^ + +Currently experimental support. If the input database includes the type ``TIMEZONE_LTZ`` (or equivalent) +then the output database must also be a Snowflake database. + +Other +^^^^^ + +In principle DataFaker can support any database that SqlAlchemy supports. +However, each new database supported so far has required significant effort, +so no guarantees can be made for, for example, MySQL or MariaDB. diff --git a/examples/omop-mssql/README.md b/examples/omop-mssql/README.md index 7c0bbfed..64662a35 100644 --- a/examples/omop-mssql/README.md +++ b/examples/omop-mssql/README.md @@ -51,8 +51,6 @@ PostgreSQL uses `SERIAL` for autoincrement columns. The code already strips `SER [datafaker/utils.py:297-309](../../datafaker/utils.py) toggles `connection.autocommit` directly on the DBAPI connection before executing `SET` commands. The availability and behaviour of `autocommit` differs between `psycopg2`, `pyodbc`, and `pymssql`, so this would need testing or an abstraction per driver. -**Deferred.** `set_db_settings` is only ever called for DuckDB connections (`parquet_dir` source feature); MS-SQL connections never reach it. Fixing the `autocommit` toggle in isolation would also leave the `SET {k} TO {v}` SQL syntax broken for MS-SQL (which requires `SET {k} {v}`). Both issues should be addressed together if `set_db_settings` is ever extended to MS-SQL. - ## Setup ### 1. Install the MS-SQL Python extras diff --git a/poetry.lock b/poetry.lock index a34332f0..ab1b9401 100644 --- a/poetry.lock +++ b/poetry.lock @@ -205,6 +205,21 @@ files = [ [package.extras] dev = ["backports.zoneinfo", "freezegun (>=1.0,<2.0)", "jinja2 (>=3.0)", "pytest (>=6.0)", "pytest-cov", "pytz", "setuptools", "tzdata"] +[[package]] +name = "backports-tarfile" +version = "1.2.0" +description = "Backport of CPython tarfile module" +optional = false +python-versions = ">=3.8" +files = [ + {file = "backports.tarfile-1.2.0-py3-none-any.whl", hash = "sha256:77e284d754527b01fb1e6fa8a1afe577858ebe4e9dad8919e34c862cb399bc34"}, + {file = "backports_tarfile-1.2.0.tar.gz", hash = "sha256:d75e02c268746e1b8144c278978b6e98e85de6ad16f8e4b0844a154557eca991"}, +] + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["jaraco.test", "pytest (!=8.0.*)", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)"] + [[package]] name = "black" version = "23.12.1" @@ -251,6 +266,44 @@ d = ["aiohttp (>=3.7.4)", "aiohttp (>=3.7.4,!=3.9.0)"] jupyter = ["ipython (>=7.8.0)", "tokenize-rt (>=3.2.0)"] uvloop = ["uvloop (>=0.15.2)"] +[[package]] +name = "boto3" +version = "1.43.106" +description = "The AWS SDK for Python (Boto3)" +optional = false +python-versions = ">=3.10" +files = [ + {file = "boto3-1.43.106-py3-none-any.whl", hash = "sha256:fdf2e304f9e8864b8613591b18b36ee995560c9666033622eda1cb9057866880"}, + {file = "boto3-1.43.106.tar.gz", hash = "sha256:c11ad4c429a983493ba10014c7af9831a455c2c0eea91c1cefff74530e480277"}, +] + +[package.dependencies] +botocore = ">=1.43.106,<1.44.0" +jmespath = ">=0.7.1,<2.0.0" +s3transfer = ">=0.19.0,<0.20.0" + +[package.extras] +crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] + +[[package]] +name = "botocore" +version = "1.43.106" +description = "Low-level, data-driven core of boto 3." +optional = false +python-versions = ">=3.10" +files = [ + {file = "botocore-1.43.106-py3-none-any.whl", hash = "sha256:c1fb8818f9957cb5037f73db34ac1a12ba43562e31d6e7f9cb7e5fc64e1dba78"}, + {file = "botocore-1.43.106.tar.gz", hash = "sha256:006870b3b4e40547232ad12c3bb4faec91bbbe0659aafaa3b7fa48a112c4ee97"}, +] + +[package.dependencies] +jmespath = ">=0.7.1,<2.0.0" +python-dateutil = ">=2.1,<3.0.0" +urllib3 = ">=1.25.4,<2.2.0 || >2.2.0,<3" + +[package.extras] +crt = ["awscrt (==0.36.0)"] + [[package]] name = "certifi" version = "2026.7.22" @@ -262,6 +315,118 @@ files = [ {file = "certifi-2026.7.22.tar.gz", hash = "sha256:741e2c3b351ddf169a738da9f2c048608ff7f2c5cc02f1ebc6b118bb090d5d55"}, ] +[[package]] +name = "cffi" +version = "2.1.1" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.10" +files = [ + {file = "cffi-2.1.1-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:baed1e86cc735622097354b9d1281406caf42ff42a886d29faa8e8d1630333be"}, + {file = "cffi-2.1.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ca82be1a1d406ecfe1d25dc16cb33488e5a16bf4438c9fb590484ea29d92478b"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:42e2f76b9455f5a9a844f770bf3e200ed3da0e15f5df3db9c31fe80b04b3d004"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:5a59cc1c4442bc3d5c703bf720b51138d0bfc173618807c9ee2490a7541dd3d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:9f8d177621de5cb38ee3e731eda45d421db093ec0739f46a5594babda7987a98"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:75f80557d1389eddbd0de2681f6a390a0c5338c31ddaa821381c203fc3fd50d9"}, + {file = "cffi-2.1.1-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:194cffa889098ced9976c3fc6340305e43f6303657d298da55366907c05c22d6"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:5bb4e7ea95dcd6a014a6fef62e62467d67d8e582326443f3d68e71d6320a9fcf"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:3d22a20b1fb1632cc72c22f95f7b0d2961c3e1c235f245ba4c606c4771035659"}, + {file = "cffi-2.1.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1dea0e4d7d4f11f619fe8c1d76caf49e24405b4b5743c0e3be16a500ecd930c9"}, + {file = "cffi-2.1.1-cp310-cp310-win32.whl", hash = "sha256:7ce713ace7c0e4520535b42b77eaa742c16dab813978064913e5a3cf82973b41"}, + {file = "cffi-2.1.1-cp310-cp310-win_amd64.whl", hash = "sha256:a48d62ab9d6f4f98c983223a547af44be6ca3691074c31cecced6facd3ba2dc1"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:c8d2c9fd1f2d16f780d15127abb050d13d1a76c03a4bd87d7e4980e45e511e12"}, + {file = "cffi-2.1.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:398aff33cee2767e3e781d2554c54bd0dff386bb437581e0d8011fde1a942ec1"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:154852545011f779917b11c78db2358d095da62a9a172b78ad0a583ee5adc0d0"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:3311ed60d36f83378794e1009ac6258bafbf81f7888b4caa7b35a521e3f95813"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:6e192623c49c94421616a5778fba35cf0d5a8d000650c1967ef4448ee5cdd990"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a6e721d4b0e45d5b65e87534470e67b18dcd092c83f68fba09f152b9cbc061af"}, + {file = "cffi-2.1.1-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:34e261f78cb6ceaaa36f42f2613f4380d94d9c759a9c73c769ee6e0247364632"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:7225e4514edb64eb6740324353e0da0711954fd8d7da4576755b1c6e09b697cd"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:df913725b79db7bcf03448f36b7bf8815363417d5b58deecf9305e3e30f0f21a"}, + {file = "cffi-2.1.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:f5cfbc5fe74540d335175b656c725d74d90e3730c626d92575eea35029d9afaa"}, + {file = "cffi-2.1.1-cp311-cp311-win32.whl", hash = "sha256:f8ec5e643a9a937f64e1999eb9f75d072263751912dc5cd06d3c85f8f44be7c3"}, + {file = "cffi-2.1.1-cp311-cp311-win_amd64.whl", hash = "sha256:42f6930c31dc7f50732c9ae793c2786c7b6b044195967bbdde40bb9be81c4cc0"}, + {file = "cffi-2.1.1-cp311-cp311-win_arm64.whl", hash = "sha256:c7659f22557c5a0bc4855cd635f55edec690cc008a40768527762cb9fb263455"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:c8c69575568085ba0b1b10c0249d779a214aea6f6522e949a0fc9fb0fcb449d0"}, + {file = "cffi-2.1.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:f81b3b8f3d4e343550fa4baa0e479bba9f2d29ce9c2e9b51d1ce1718d7442fcf"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:811bd1e21d32de12efca32393a0ab3f5133b54fce9bd44b8bd77ab07da14bf6a"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:68e62fe11f30d5ca8289242866f0a5291402d8529ca2178ab8afc5c9694ae890"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4a7c934f7360e8cd64fe9efadcbd10c7c6364f531e432b9a4bf5ccbc9e0e8b50"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:3143d81e29e1e20a9ce10901ec369012947876596f75a222235965f2b7ae832e"}, + {file = "cffi-2.1.1-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c1453022f490d2459a11819d83ad1d586e9ff65a12ac3e705ffebd46d3685dcf"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:208f941bb9d18e768138677f0a6d2ce01f590df56043dda1df1535ac57c88517"}, + {file = "cffi-2.1.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:210019b6c7cf07f081b4c54635c8cf744377001350e29cc0f81c4377b4797735"}, + {file = "cffi-2.1.1-cp312-cp312-win32.whl", hash = "sha256:046bfc24911b37851ee1b51aab8bffe713d89c68c6a057b09484ce9fd5f69b4e"}, + {file = "cffi-2.1.1-cp312-cp312-win_amd64.whl", hash = "sha256:f53e442b08449d42821fa4a4fba000095af9f62742a500f978a9f557ec44339a"}, + {file = "cffi-2.1.1-cp312-cp312-win_arm64.whl", hash = "sha256:7bde5e4cc5c10140859842b9d383af292b22639a4dffb725314baf45968cef80"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:b5bdfd1c873d4e093aabc0ca84c4ca6dbc4f752afb5c86f146d9742580c9da2e"}, + {file = "cffi-2.1.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:31348097ff5bbe827ccc41795d4dd099d9f0625e7def00ee653c137a490c2a6c"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_10_15_x86_64.whl", hash = "sha256:9d2055050ea716bd38b7f7f1579c275386646b4894c155a3e2f3cd62ed41b7c6"}, + {file = "cffi-2.1.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:19ee6127ee34de7d83ce3d371ebc5ed91addbdcc39f9ab15ce4eb35a4e534971"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux1_i686.manylinux2014_i686.manylinux_2_17_i686.manylinux_2_5_i686.whl", hash = "sha256:6a8dddef476fab96d066d578fc88526767b836ab5ab21754e1d5bf3879c31c7c"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:f16c709686a78c727bbbf059f92b0bf41c6fc60deec706d2dc19f529175a6125"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:fcd22650c908d7b7da162bbfaab594a1227a15d1643a98c68b122ac642fa2264"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:aa9511c62d14da7aacc9b4bf51f3f697a621e83b2d6919008243c3aad168eea3"}, + {file = "cffi-2.1.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:a931079504ecc49efed7744c476a5c343a92fabf66dec2db95edb1b2fdc770e2"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:a2d7755bef5a12ed488f4ef1f1b69ee9191d7396083b755a5d2295f6edb4768b"}, + {file = "cffi-2.1.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e0bcb7e0f677f543555d2adff3bf19c05f66cdb4796e5ff602442ab2fe3c4ef7"}, + {file = "cffi-2.1.1-cp313-cp313-win32.whl", hash = "sha256:334644fbac4eff73d985a17a91226df55d0f394160c4cfb880e084c8f7161cac"}, + {file = "cffi-2.1.1-cp313-cp313-win_amd64.whl", hash = "sha256:1aa5645c30469b09530c4ebca77ebf8f17618293c58f8549cb1a543a50236e7d"}, + {file = "cffi-2.1.1-cp313-cp313-win_arm64.whl", hash = "sha256:63bbfd5ded17c4840ac07cd8f1c21ba9d9708141f840b324f422f41b207e3973"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphoneos.whl", hash = "sha256:7dbb61fe3a7699468030f71bbe5f8a0e326a151daa91beb11a6fc1f980c55e1c"}, + {file = "cffi-2.1.1-cp314-cp314-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:f24fb43132a4c6b4cb4eb029492919b2db645be6808d738f244fd146c03c32cb"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:d28630f5854ab07ab1fd4aba756de52326c82e6be15d414b12793f1975048b54"}, + {file = "cffi-2.1.1-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:661c298b4821edebead0c91edd2b00374d67ad7c5a1f7a91d4442633b79d6a72"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:58acb8ab8e295e6c5ea12f888cbb13cf21511ef2a3303a23f4325c29d17fe5c1"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:456a61fa52d579ebf9df2e9552ead5129855dbaff6c1e5a9b1bc408809bdc062"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a4f00aa42f75d6e4595e8866e748cc1705adc0cddfeb2ca86d0d03993d63ba03"}, + {file = "cffi-2.1.1-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:b0431303acaea1089ad4b3e9ce4e6518193def1118d4073ca848635ee4ea2e96"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:64faea20f4e2613363a1a9b9c7dd73058f3ecd00133a511e72ad7c511658f527"}, + {file = "cffi-2.1.1-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:5c58fe613dc5e5336357eff555824a314d8e43282600435c8d1cb6a7a2fedd13"}, + {file = "cffi-2.1.1-cp314-cp314-win32.whl", hash = "sha256:1a18a57b58cfb21fc28d72e876acf10eaed67a1ed96226f92af4df681d571c4c"}, + {file = "cffi-2.1.1-cp314-cp314-win_amd64.whl", hash = "sha256:3222ba5d678f80a030e6afbcc33dc1ae5cb45facabb61cee2c7016b8432fde48"}, + {file = "cffi-2.1.1-cp314-cp314-win_arm64.whl", hash = "sha256:ab36d55f9ed2d067327667c2fea18dda018eb628dd6347aa01dda6cf1f5d3836"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_10_15_x86_64.whl", hash = "sha256:7750c6449dff7864bb9bb27ddfb0267756189201a3afc911d82b3caacd70dfc3"}, + {file = "cffi-2.1.1-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:0beceaabe56af686895136a2de78db54ecd8e4046b236b8fd6d6cb61389e9bf2"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:49cbc70e6542d4ccccb936558d1064a8012541e78f821f955cff24e357776c94"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:e2d65b31f36619cda3999b78b2aa9632e76b78448e7a56fc4240824200e7c4fc"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:28907ab9bfb6aa13184cfc17c6b8e1023c5ab6fd7076d8c20a35e59fe04f8f29"}, + {file = "cffi-2.1.1-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:51b31d1c98274844cfd7838ce00bfc27c7423a4dc00fc0772fc3331c2cc90676"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:5e7cecbaadb83884793e05828cee59b210b24583b9c7425d0ba6a754fe22eb4e"}, + {file = "cffi-2.1.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:25792eac27877609e7bb06d42ff88278a6624fff2ba9bbb523c09616b117e80f"}, + {file = "cffi-2.1.1-cp314-cp314t-win32.whl", hash = "sha256:8ef53b2de9bcb9197d31854256575d59dbac0cba72ac627bb291ef5eceb74be4"}, + {file = "cffi-2.1.1-cp314-cp314t-win_amd64.whl", hash = "sha256:616f097f2fe415bc92a247f02e11f634e1f9e9a83d327e3c915c15089c87869e"}, + {file = "cffi-2.1.1-cp314-cp314t-win_arm64.whl", hash = "sha256:ad2c86c495b899d862ea0f4b42891b8713a3bd45dd4105c7fd51c2a72f39f3a5"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphoneos.whl", hash = "sha256:dddad92b554513a31f272570678ba307fb9f618f05e3d4a5eacafff9eae03e1d"}, + {file = "cffi-2.1.1-cp315-cp315-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:da0e573f9f97159390c89d9f1a9e41908b66d408cc5b58d08cf3847d844c531b"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_10_15_x86_64.whl", hash = "sha256:fb92203a88b3d3053034db775110081c49d28be6551923805e039924093761e4"}, + {file = "cffi-2.1.1-cp315-cp315-macosx_11_0_arm64.whl", hash = "sha256:2ae64be792b8966f2c69538199728b290e34726562896df1e5dc8ffd8d8188e8"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:507a24c282e0f42f8ed737cf048572cbf580468da5555764a8331735e9c736b6"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:246fa40ce8645a614ff682e0b70f37134e460eaf93a775e0cbe3cca585a67a80"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:471cee653ae88de62096552e6d24ccb4a5adb8c8c9f10b5054d0122c15bf2779"}, + {file = "cffi-2.1.1-cp315-cp315-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:aeae0e330c9f6acd681f647d46cefd30c29f93e3392882e792e82080c9691399"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_aarch64.whl", hash = "sha256:42a494cee34437f05546455144f2b5d9ac09b1face62bcfce597d2e521066688"}, + {file = "cffi-2.1.1-cp315-cp315-musllinux_1_2_x86_64.whl", hash = "sha256:cc572dace3f60ef98d7b12ff411d20f5362feb31a0439eab0085bbfd349982d7"}, + {file = "cffi-2.1.1-cp315-cp315-win32.whl", hash = "sha256:4f42141fc14250de6dde5ee7ea4432be017252d91f19c5ad043c084cea629cac"}, + {file = "cffi-2.1.1-cp315-cp315-win_amd64.whl", hash = "sha256:e6e8cff14d6fb0be70a09c0bdc58096f501952d04624ebf867e0e56da2df8960"}, + {file = "cffi-2.1.1-cp315-cp315-win_arm64.whl", hash = "sha256:27350daa11d4f10c540e6e89dada4c54feb7256ad03e9a4dc075ebad7ba360d1"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_10_15_x86_64.whl", hash = "sha256:c26608d2222fb1e94487e4a387d85f13eb55d5ed725cb25a0c589ac4ee60e7bc"}, + {file = "cffi-2.1.1-cp315-cp315t-macosx_11_0_arm64.whl", hash = "sha256:4be96343e422f2dfcd12ab5c9f5aebe03f82f737c6bffeca6830b3875cb44aab"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:937c0052c05a31ca1daf18de3158eed4dbfcb9cc107adbea227728d647be701e"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:df423d40ee8654634421812bc3b196da3f9bd7d32929da813f8394c4348a5358"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:a730a083190634c65cca36ba5f489531576ebd79bcd5c8e172130f6453127231"}, + {file = "cffi-2.1.1-cp315-cp315t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:363e05fa78e15116c3c32c210ee36884fd6b9afa6d440e47112c3bd511d64cb6"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_aarch64.whl", hash = "sha256:770de9db11e84213beec501cfcaa013b019820ca881e03344dea5844f7876d94"}, + {file = "cffi-2.1.1-cp315-cp315t-musllinux_1_2_x86_64.whl", hash = "sha256:7da0c5eff80f0197f3b3d1232ec5a682a9325f4ae9016a78f5f5ca35f9ced1f5"}, + {file = "cffi-2.1.1-cp315-cp315t-win32.whl", hash = "sha256:06c72bb76605a4b0cd0aad6930b69d4baf7dd5d806cfc409b824191099700e66"}, + {file = "cffi-2.1.1-cp315-cp315t-win_amd64.whl", hash = "sha256:d9c275eaacd24aa73f94ffd6de08fc3f932424d8b6c376f4bed7cde376fe7bc3"}, + {file = "cffi-2.1.1-cp315-cp315t-win_arm64.whl", hash = "sha256:d18e5ac0f2f03f4f518d3e23db0f0cad7faa1da8620e9c09461d443bbf6e6692"}, + {file = "cffi-2.1.1.tar.gz", hash = "sha256:dd31f52ea1086513bb9df30f8fcee9b8918323ae067a3d5b78bc826a000712be"}, +] + +[package.dependencies] +pycparser = {version = "*", markers = "implementation_name != \"PyPy\""} + [[package]] name = "cfgv" version = "3.5.0" @@ -545,6 +710,81 @@ files = [ [package.extras] dev = ["black (==22.3.0)", "hypothesis (==6.60.0)", "numpy", "pytest (>=5.30)", "pytest-benchmark", "pytest-xdist"] +[[package]] +name = "cryptography" +version = "50.0.2" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = "!=3.9.0,!=3.9.1,>=3.9" +files = [ + {file = "cryptography-50.0.2-cp311-abi3-macosx_11_0_arm64.whl", hash = "sha256:fa8f5efb344d6908a1ce62f4a24e2e5780f825d6f53f5f50ec5ffacac72936cb"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:79def8d059362e7831389ed3be0ecdf58a89386e1271e35dd9f5af84e81bffd0"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:630ebfea3bf689d075f82316324ff7433dc447fe6bc1bfc76524b74b4a9567d2"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:f9f6143a8c75945eb960d9eb98905a441394abfa24afaae239d514ffb2586480"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:a582ab2ae1d34f67112cadc86702774c9ea4374df6bca6afe672817203c99134"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:4061c0079120205fb760c58acab6443e217307dcf05e3702cf970e0689972856"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:ac9ed99d81760c62fe89d5f0815cdfa1ba9a35141cf30f1c2d044f04b4803d2e"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:87e9ce85beb6b328ba370cc6e6aea483c92617b4c95b1d33a49297eb662bfb04"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:f265528741e048bce55c3463ed721fb0aa45a5888d8add8cfeccb3035451bbdc"}, + {file = "cryptography-50.0.2-cp311-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:9dab55f57c74c3cad24c323bacbbd04be4705ba6eb0d92e920b1fc4837ed5079"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:25784ce8b9621c90c643efb9e1e2162ab3b0224cae446ad5e70e7fcb1ce18b51"}, + {file = "cryptography-50.0.2-cp311-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:85d0d9a31b9098e98534226d5686b47264b95e62ce459dc2e62fdfc809f9fe93"}, + {file = "cryptography-50.0.2-cp311-abi3-win_amd64.whl", hash = "sha256:7afa5a6602a9f29af1f3a2965f831bae7c9d5d597b7cbb716d41ab3b7d89879c"}, + {file = "cryptography-50.0.2-cp314-cp314t-macosx_11_0_arm64.whl", hash = "sha256:f785f6161f202ab04d8ca194158968798e480ca058943907972da5f12e2881e8"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:0ecbc5652bdb6fc9eaf89a7d196e20941adfe812f43bc4ca05d9150496821047"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ab50ee449bf968271e820086f10a33d101dd060370abc10bcd22279be2656539"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_aarch64.whl", hash = "sha256:a9f7355e6fab51f6c369b86fb7571cffa05edee2c2121e0380a37fb9ac1cd5c1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_ppc64le.whl", hash = "sha256:94e5e9f108ee10471288214d3d233fbfbb492840a8457eb85178d643ddeb32c7"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_28_x86_64.whl", hash = "sha256:241449bf940a5d27309bd317e6f9a2af6932113818bb2b8f5c59ddc7ef16da18"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_31_armv7l.whl", hash = "sha256:d8947001be83df1394050758ce0e745dd74fb134eef0a4b5124208dfc3a68c37"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_aarch64.whl", hash = "sha256:4a20ce1e5cb4284a86692fdcba7cb8754185c6b2e5c56fcef3751cf451d3cdc2"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_ppc64le.whl", hash = "sha256:84f964e537f916e2cc85199e5a88742e964939b575ac8598b3f9d6cc416cdaf1"}, + {file = "cryptography-50.0.2-cp314-cp314t-manylinux_2_34_x86_64.whl", hash = "sha256:828d49b0ff5a0e3975865571c5d91dbbdd0d38d8289b249a163e9425413a5e05"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_aarch64.whl", hash = "sha256:deb9fde5c60e437ee4821bc9bc39ff31b42135c27e1dc61ef0a629389c1de62e"}, + {file = "cryptography-50.0.2-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:8c71ba2cd31fc93748c38e1b613200ff1c2665cbfd5341fe3a61cfde35a1430e"}, + {file = "cryptography-50.0.2-cp314-cp314t-win_amd64.whl", hash = "sha256:78198641e5be9521beea5aa782bb551a58068d10e6eb04c9c680c1b69f2e7d45"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-macosx_11_0_arm64.whl", hash = "sha256:edc3342adf8f697fc5f59c887a304356f147b397809440ed64e2fa6af2f50f37"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:d370b8d1dfcdf7130178137f6fbee6140774a1acc6cacefc4b42643ec11d0a3a"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:f2f9bd7f90c64fe89253f0a2c05e3c4856072660429ce8831b4235bf29403a67"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_aarch64.whl", hash = "sha256:e275096ea1e60cc595cda2836fd4a6c725d1125108b868be17f53684d164e2cc"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_ppc64le.whl", hash = "sha256:b13478603dcd0a2479ff8e87e2c19a7d525734686fe3c49542472293a204212d"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_28_x86_64.whl", hash = "sha256:58a0c478eeca76fe5e07993c5a0703def34a6dc6a0cda4f5564639b33112ffe7"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_31_armv7l.whl", hash = "sha256:d38cdff612d06fa6a32840d5e1b1f7a27cee4a349aa9085d94a67789d6bfd408"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_aarch64.whl", hash = "sha256:fdd28f912fccfec1846a94e2e1e8f9b0012f557f0c46fe4f3eb0d7a87afcf90b"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_ppc64le.whl", hash = "sha256:cbc8738fd8526d80f35cb3a40d41f41a2e7030bb3b18b09a6778ef63d291c2fd"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-manylinux_2_34_x86_64.whl", hash = "sha256:e105ab60406787da31fccc883fc0f733af1efd78f0136a4599692c4083a73d0c"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_aarch64.whl", hash = "sha256:6f8700550aa1474a91e5dc07049c46f98b423b5b1ddd0483e0b51362eeeaf5be"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-musllinux_1_2_x86_64.whl", hash = "sha256:c71be1cbfa5cd9a41ee452acf1eccd82b2c05950358b106ec8ceb83411d1a020"}, + {file = "cryptography-50.0.2-cp315-abi3.abi3t-win_amd64.whl", hash = "sha256:c423ab384a46c4dff7217b2ea5ba2e11cffdeab6441acd04cf65a369caf0366c"}, + {file = "cryptography-50.0.2-cp39-abi3-macosx_11_0_arm64.whl", hash = "sha256:0ec5f09541743261e66e291b4a0cbf0fb2997aeaab6d9e9c740b9dba1b58d1c2"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c5e67125c7dca78d199ec4e116aa93dbb83494808ecbb8211a2cb09b1bf41dbd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:ee247f5c245c9a2fe7c8e2214e295918838e44e00a45a6718451e4004219e767"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:dfe9763530994147d9af1def057a5b9658b00e8f8fe8743d144d1e0911c2e454"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_ppc64le.whl", hash = "sha256:58ddb5a8e3179d12f19e4ea34d2d32e9d63a4baa142c875c1eb59f41b7243acd"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:f21e8a22c8605750c7af886bab299a363721264061b4ac0a30efb73cfd58efc5"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_31_armv7l.whl", hash = "sha256:9c8402a82ea0dc4ceeab793db05f0fafa8ca139ca34fcde5df0f596103c74107"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_aarch64.whl", hash = "sha256:0ddc924c04591c2811ca024d62ecad4f7f6f08af8939c211438f48a16bd23602"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_ppc64le.whl", hash = "sha256:a6557e5f38e065ca9fbdaf7cfc7435ecb1d113aa81a022d1b51921ee7432e227"}, + {file = "cryptography-50.0.2-cp39-abi3-manylinux_2_34_x86_64.whl", hash = "sha256:1981f1db4630889b9ef7803fadef12b056f428cb6b85c27ba57b774793b6093c"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:7a8701d6b584d76e909e3d305b7d126b41439876a5aaf76cddc67fc230eafa2e"}, + {file = "cryptography-50.0.2-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:ce47f66801c20ec6c6632453bb5960fe38939e9306970b48b3a5a26de7745d94"}, + {file = "cryptography-50.0.2-cp39-abi3-win_amd64.whl", hash = "sha256:4e81d95e5bafc2d6e34e4bed780e53e4d5b9a2f928573428aa4d35fbec1eb0de"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:92e665960f25fcdc73725b9cec7a3824f279ba97a98653afe9ffac2e43668f67"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:eef4c2f3423810b3070ab391f85436d2f8bbfcb286ac15cbc73190b3563b1f1a"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_aarch64.whl", hash = "sha256:7c6d0330c472d96f6a6afe24d80dfdf15176c33096f0a4397ae4c60f3dd3be48"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp73-manylinux_2_34_x86_64.whl", hash = "sha256:1ba34f04897fcdaa73f74145c25f3ec146fbd56593853e88adc2e811303c5f42"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-macosx_11_0_arm64.whl", hash = "sha256:3dc4fd8058cea1644971207d530e1a03a184a805ffc8ebdddf0599d78a331b81"}, + {file = "cryptography-50.0.2-pp311-pypy311_pp80-win_amd64.whl", hash = "sha256:7b75de3c8b3be1cdb1052747c929440c3eea46c1bc2cb8a6e3a48388e9b7b452"}, + {file = "cryptography-50.0.2.tar.gz", hash = "sha256:7b46165bb56eb4704e2eaaf86f3c940d19154535d9b0ca7d6d590b04060e00d5"}, +] + +[package.dependencies] +cffi = {version = ">=2.0.0", markers = "platform_python_implementation != \"PyPy\""} +typing-extensions = {version = ">=4.13.2", markers = "python_full_version < \"3.11\""} + +[package.extras] +ssh = ["bcrypt (>=3.1.5)"] + [[package]] name = "dataclasses-json" version = "0.6.7" @@ -955,6 +1195,29 @@ files = [ {file = "imagesize-2.0.0.tar.gz", hash = "sha256:8e8358c4a05c304f1fccf7ff96f036e7243a189e9e42e90851993c558cfe9ee3"}, ] +[[package]] +name = "importlib-metadata" +version = "9.0.1" +description = "Read metadata from Python packages" +optional = false +python-versions = ">=3.10" +files = [ + {file = "importlib_metadata-9.0.1-py3-none-any.whl", hash = "sha256:bba5600596a7e21f3eef53281cf28d6a5195634d2f2b78ff9501a3272c6eaab0"}, + {file = "importlib_metadata-9.0.1.tar.gz", hash = "sha256:ab830580bc0ef3db61ce8fae716389e5462b67e033018bab6d8f80ef17172f99"}, +] + +[package.dependencies] +zipp = ">=3.20" + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +perf = ["ipython"] +test = ["packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-perf (>=0.17)"] +type = ["pytest-mypy (>=1.0.1)"] + [[package]] name = "iniconfig" version = "2.3.0" @@ -980,6 +1243,83 @@ files = [ [package.extras] colors = ["colorama (>=0.4.6)"] +[[package]] +name = "jaraco-classes" +version = "3.4.0" +description = "Utility functions for Python class constructs" +optional = false +python-versions = ">=3.8" +files = [ + {file = "jaraco.classes-3.4.0-py3-none-any.whl", hash = "sha256:f662826b6bed8cace05e7ff873ce0f9283b5c924470fe664fff1c2f00f581790"}, + {file = "jaraco.classes-3.4.0.tar.gz", hash = "sha256:47a024b51d0239c0dd8c8540c6c7f484be3b8fcf0b2d85c13825780d3b3f3acd"}, +] + +[package.dependencies] +more-itertools = "*" + +[package.extras] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[[package]] +name = "jaraco-context" +version = "6.1.2" +description = "Useful decorators and context managers" +optional = false +python-versions = ">=3.10" +files = [ + {file = "jaraco_context-6.1.2-py3-none-any.whl", hash = "sha256:bf8150b79a2d5d91ae48629d8b427a8f7ba0e1097dd6202a9059f29a36379535"}, + {file = "jaraco_context-6.1.2.tar.gz", hash = "sha256:f1a6c9d391e661cc5b8d39861ff077a7dc24dc23833ccee564b234b81c82dfe3"}, +] + +[package.dependencies] +"backports.tarfile" = {version = "*", markers = "python_version < \"3.12\""} + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["jaraco.test (>=5.6.0)", "portend", "pytest (>=6,!=8.1.*)"] +type = ["pytest-mypy (>=1.0.1)"] + +[[package]] +name = "jaraco-functools" +version = "4.6.0" +description = "Functools like those found in stdlib" +optional = false +python-versions = ">=3.10" +files = [ + {file = "jaraco_functools-4.6.0-py3-none-any.whl", hash = "sha256:99e3dc0060c5cbe8fcd1cdb36258e2a65ca40f1566b2033b12abb1bb44dd3c30"}, + {file = "jaraco_functools-4.6.0.tar.gz", hash = "sha256:880c577ec9720b3a052d5bc611fb9f2269b3d87902ef42440df443b88e443280"}, +] + +[package.dependencies] +more_itertools = "*" + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["jaraco.classes", "pytest (>=6,!=8.1.*)"] +type = ["pytest-mypy (>=1.0.1)"] + +[[package]] +name = "jeepney" +version = "0.9.0" +description = "Low-level, pure Python DBus protocol wrapper." +optional = false +python-versions = ">=3.7" +files = [ + {file = "jeepney-0.9.0-py3-none-any.whl", hash = "sha256:97e5714520c16fc0a45695e5365a2e11b81ea79bba796e26f9f1d178cb182683"}, + {file = "jeepney-0.9.0.tar.gz", hash = "sha256:cf0e9e845622b81e4a28df94c40345400256ec608d0e55bb8a3feaa9163f5732"}, +] + +[package.extras] +test = ["async-timeout", "pytest", "pytest-asyncio (>=0.17)", "pytest-trio", "testpath", "trio"] +trio = ["trio"] + [[package]] name = "jinja2" version = "3.1.6" @@ -997,6 +1337,17 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jmespath" +version = "1.1.0" +description = "JSON Matching Expressions" +optional = false +python-versions = ">=3.9" +files = [ + {file = "jmespath-1.1.0-py3-none-any.whl", hash = "sha256:a5663118de4908c91729bea0acadca56526eb2698e83de10cd116ae0f4e97c64"}, + {file = "jmespath-1.1.0.tar.gz", hash = "sha256:472c87d80f36026ae83c6ddd0f1d05d4e510134ed462851fd5f754c8c3cbb88d"}, +] + [[package]] name = "json-schema-for-humans" version = "1.5.1" @@ -1053,6 +1404,35 @@ files = [ [package.dependencies] referencing = ">=0.31.0" +[[package]] +name = "keyring" +version = "25.7.0" +description = "Store and access your passwords safely." +optional = false +python-versions = ">=3.9" +files = [ + {file = "keyring-25.7.0-py3-none-any.whl", hash = "sha256:be4a0b195f149690c166e850609a477c532ddbfbaed96a404d4e43f8d5e2689f"}, + {file = "keyring-25.7.0.tar.gz", hash = "sha256:fe01bd85eb3f8fb3dd0405defdeac9a5b4f6f0439edbb3149577f244a2e8245b"}, +] + +[package.dependencies] +importlib_metadata = {version = ">=4.11.4", markers = "python_version < \"3.12\""} +"jaraco.classes" = "*" +"jaraco.context" = "*" +"jaraco.functools" = "*" +jeepney = {version = ">=0.4.2", markers = "sys_platform == \"linux\""} +pywin32-ctypes = {version = ">=0.2.0", markers = "sys_platform == \"win32\""} +SecretStorage = {version = ">=3.2", markers = "sys_platform == \"linux\""} + +[package.extras] +check = ["pytest-checkdocs (>=2.4)", "pytest-ruff (>=0.2.1)"] +completion = ["shtab (>=1.1.0)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["pyfakefs", "pytest (>=6,!=8.1.*)"] +type = ["pygobject-stubs", "pytest-mypy (>=1.0.1)", "shtab", "types-pywin32"] + [[package]] name = "librt" version = "0.13.0" @@ -1348,6 +1728,17 @@ files = [ factory = ["factory-boy (>=3.3.0,<4.0.0)"] pytest = ["pytest (>=7.2,<8.0)"] +[[package]] +name = "more-itertools" +version = "11.1.0" +description = "More routines for operating on iterables, beyond itertools" +optional = false +python-versions = ">=3.10" +files = [ + {file = "more_itertools-11.1.0-py3-none-any.whl", hash = "sha256:4b65538ae22f6fed0ce4874efd317463a7489796a0939fa66824dd542125a192"}, + {file = "more_itertools-11.1.0.tar.gz", hash = "sha256:48e8f4d9e7e5878571ecf6f2b4e57634f93cd474cc8cfbd2376f2d11b396e30d"}, +] + [[package]] name = "mypy" version = "2.3.0" @@ -1757,79 +2148,30 @@ wcwidth = ">=0.3.5" tests = ["pytest (>=9)", "pytest-cov", "pytest-lazy-fixtures"] [[package]] -name = "psycopg2-binary" -version = "2.9.12" +name = "psycopg2" +version = "2.9.13" description = "psycopg2 - Python-PostgreSQL Database Adapter" optional = false -python-versions = ">=3.9" +python-versions = ">=3.10" files = [ - {file = "psycopg2_binary-2.9.12-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9b818ceff717f98851a64bffd4c5eb5b3059ae280276dcecc52ac658dcf006a4"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d2fa0d7caca8635c56e373055094eeda3208d901d55dd0ff5abc1d4e47f82b56"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:864c261b3690e1207d14bbfe0a61e27567981b80c47a778561e49f676f7ce433"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c5ee5213445dd45312459029b8c4c0a695461eb517b753d2582315bd07995f5e"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:6f9cae1f848779b5b01f417e762c40d026ea93eb0648249a604728cda991dde3"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:63a3ebbd543d3d1eda088ac99164e8c5bac15293ee91f20281fd17d050aee1c4"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:d6fcbba8c9fed08a73b8ac61ea79e4821e45b1e92bb466230c5e746bbf3d5256"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_ppc64le.whl", hash = "sha256:36512911ebb2b60a0c3e44d0bb5048c1980aced91235d133b7874f3d1d93487c"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_riscv64.whl", hash = "sha256:8ffdb59fe88f99589e34354a130217aa1fd2d615612402d6edc8b3dbc7a44463"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:a46fe069b65255df410f856d842bc235f90e22ffdf532dda625fd4213d3fd9b1"}, - {file = "psycopg2_binary-2.9.12-cp310-cp310-win_amd64.whl", hash = "sha256:ab29414b25dcb698bf26bf213e3348abdcd07bbd5de032a5bec15bd75b298b03"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5c8ce6c61bd1b1f6b9c24ee32211599f6166af2c55abb19456090a21fd16554b"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:b4a9eaa6e7f4ff91bec10aa3fb296878e75187bced5cc4bafe17dc40915e1326"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:c6528cefc8e50fcc6f4a107e27a672058b36cc5736d665476aeb413ba88dbb06"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:e4e184b1fb6072bf05388aa41c697e1b2d01b3473f107e7ec44f186a32cfd0b8"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:4766ab678563054d3f1d064a4db19cc4b5f9e3a8d9018592a8285cf200c248f3"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:5a0253224780c978746cb9be55a946bcdaf40fe3519c0f622924cdabdafe2c39"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:0dc9228d47c46bda253d2ecd6bb93b56a9f2d7ad33b684a1fa3622bf74ffe30c"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_ppc64le.whl", hash = "sha256:f921f3cd87035ef7df233383011d7a53ea1d346224752c1385f1edfd790ceb6a"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_riscv64.whl", hash = "sha256:3d999bd982a723113c1a45b55a7a6a90d64d0ed2278020ed625c490ff7bef96c"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:29d4d134bd0ab46ffb04e94aa3c5fa3ef582e9026609165e2f758ff76fc3a3be"}, - {file = "psycopg2_binary-2.9.12-cp311-cp311-win_amd64.whl", hash = "sha256:cb4a1dacdd48077150dc762a9e5ddbf32c256d66cb46f80839391aa458774936"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5cdc05117180c5fa9c40eea8ea559ce64d73824c39d928b7da9fb5f6a9392433"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:d3227a3bc228c10d21011a99245edca923e4e8bf461857e869a507d9a41fe9f6"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:995ce929eede89db6254b50827e2b7fd61e50d11f0b116b29fffe4a2e53c4580"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:9fe06d93e72f1c048e731a2e3e7854a5bfaa58fc736068df90b352cefe66f03f"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:40e7b28b63aaf737cb3a1edc3a9bbc9a9f4ad3dcb7152e8c1130e4050eddcb7d"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:89d19a9f7899e8eb0656a2b3a08e0da04c720a06db6e0033eab5928aabe60fa9"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:612b965daee295ae2da8f8218ce1d274645dc76ef3f1abf6a0a94fd57eff876d"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_ppc64le.whl", hash = "sha256:b9a339b79d37c1b45f3235265f07cdeb0cb5ad7acd2ac7720a5920989c17c24e"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_riscv64.whl", hash = "sha256:3471336e1acfd9c7fe507b8bad5af9317b6a89294f9eb37bd9a030bb7bebcdc6"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:7af18183109e23502c8b2ae7f6926c0882766f35b5175a4cd737ad825e4d7a1b"}, - {file = "psycopg2_binary-2.9.12-cp312-cp312-win_amd64.whl", hash = "sha256:398fcd4db988c7d7d3713e2b8e18939776fd3fb447052daae4f24fa39daede4c"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:7c729a73c7b1b84de3582f73cdd27d905121dc2c531f3d9a3c32a3011033b965"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:4413d0caef93c5cf50b96863df4c2efe8c269bf2267df353225595e7e15e8df7"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:4dfcf8e45ebb0c663be34a3442f65e17311f3367089cd4e5e3a3e8e62c978777"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:c41321a14dd74aceb6a9a643b9253a334521babfa763fa873e33d89cfa122fb5"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:83946ba43979ebfdc99a3cd0ee775c89f221df026984ba19d46133d8d75d3cd9"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:411e85815652d13560fbe731878daa5d92378c4995a22302071890ec3397d019"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1c8ad4c08e00f7679559eaed7aff1edfffc60c086b976f93972f686384a95e2c"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:00814e40fa23c2b37ef0a1e3c749d89982c73a9cb5046137f0752a22d432e82f"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_riscv64.whl", hash = "sha256:98062447aebc20ed20add1f547a364fd0ef8933640d5372ff1873f8deb9b61be"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:66a7685d7e548f10fb4ce32fb01a7b7f4aa702134de92a292c7bd9e0d3dbd290"}, - {file = "psycopg2_binary-2.9.12-cp313-cp313-win_amd64.whl", hash = "sha256:b6937f5fe4e180aeee87de907a2fa982ded6f7f15d7218f78a083e4e1d68f2a0"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-macosx_10_15_x86_64.whl", hash = "sha256:6f3b3de8a74ef8db215f22edffb19e32dc6fa41340456de7ec99efdc8a7b3ec2"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-macosx_11_0_arm64.whl", hash = "sha256:1006fb62f0f0bc5ce256a832356c6262e91be43f5e4eb15b5eaf38079464caf2"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:840066105706cd2eb29b9a1c2329620056582a4bf3e8169dec5c447042d0869f"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:863f5d12241ebe1c76a72a04c2113b6dc905f90b9cef0e9be0efd994affd9354"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:a99eaab34a9010f1a086b126de467466620a750634d114d20455f3a824aae033"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:ffdd7dc5463ccd61845ac37b7012d0f35a1548df9febe14f8dd549be4a0bc81e"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_aarch64.whl", hash = "sha256:54a0dfecab1b48731f934e06139dfe11e24219fb6d0ceb32177cf0375f14c7b5"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_ppc64le.whl", hash = "sha256:96937c9c5d891f772430f418a7a8b4691a90c3e6b93cf72b5bd7cad8cbca32a5"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_riscv64.whl", hash = "sha256:77b348775efd4cdab410ec6609d81ccecd1139c90265fa583a7255c8064bc03d"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-musllinux_1_2_x86_64.whl", hash = "sha256:527e6342b3e44c2f0544f6b8e927d60de7f163f5723b8f1dfa7d2a84298738cd"}, - {file = "psycopg2_binary-2.9.12-cp314-cp314-win_amd64.whl", hash = "sha256:f12ae41fcafadb39b2785e64a40f9db05d6de2ac114077457e0e7c597f3af980"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:ee2d84ef5eb6c04702d2e9c372ad557fb027f26a5d82804f749dfb14c7fdd2ab"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:cfa2517c94ea3af6deb46f81e1bbd884faa63e28481eb2f889989dd8d95e5f03"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:ba3df2fc42a1cfa45b72cf096d4acb2b885937eedc61461081d53538d4a82a86"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:718e1fc18edf573b02cb8aea868de8d8d33f99ce9620206aa9144b67b0985e94"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-manylinux_2_27_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:5c7cb4cbf894a1d36c720d713de507952c7c58f66d30834708f03dbe5c822ccf"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-manylinux_2_38_riscv64.manylinux_2_39_riscv64.whl", hash = "sha256:049366c6d884bdcd65d66e6ca1fdbebe670b56c6c9ba46f164e6667e90881964"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:fb1828cf3da68f99e45ebce1355d65d2d12b6a78fb5dfb16247aad6bdef5f5d2"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-musllinux_1_2_ppc64le.whl", hash = "sha256:127467c6e476dd876634f17c3d870530e73ff454ff99bff73d36e80af28e1115"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-musllinux_1_2_riscv64.whl", hash = "sha256:ace94261f43850e9e79f6c56636c5e0147978ab79eda5e5e5ebf13ae146fc8fe"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:a7e39a65b7d2a20e4ba2e0aaad1960b61cc2888d6ab047769f8347bd3c9ad915"}, - {file = "psycopg2_binary-2.9.12-cp39-cp39-win_amd64.whl", hash = "sha256:f625abb7020e4af3432d95342daa1aa0db3fa369eed19807aa596367ba791b10"}, - {file = "psycopg2_binary-2.9.12.tar.gz", hash = "sha256:5ac9444edc768c02a6b6a591f070b8aae28ff3a99be57560ac996001580f294c"}, + {file = "psycopg2-2.9.13-cp310-cp310-win_amd64.whl", hash = "sha256:7d48416f6a4823ada9b33771085331b842b553df88435701bff5ddb4469905de"}, + {file = "psycopg2-2.9.13-cp311-cp311-win_amd64.whl", hash = "sha256:d16e7a5f5e400ac51ca953d42255804eff6c8a9650b1a2074f6ca6261d740382"}, + {file = "psycopg2-2.9.13-cp312-cp312-win_amd64.whl", hash = "sha256:10f7408b34412e8c0d4f8b1565541f1d651b1d00447857e5d8561b38f5c1a738"}, + {file = "psycopg2-2.9.13-cp313-cp313-win_amd64.whl", hash = "sha256:165e25c1b0e616a1f28080c5c68bd2dc015051d83c90240b2171d3e76ca2b5ff"}, + {file = "psycopg2-2.9.13-cp314-cp314-win_amd64.whl", hash = "sha256:a6f54fd8e0024f35240866b5dfff9ead2a0dbd33b8096eec438bc6093412842d"}, + {file = "psycopg2-2.9.13-cp315-cp315-win_amd64.whl", hash = "sha256:0d2fc7eedfaca0586dcf1476454598428d0d8471d3b5cb55f92015a5f9d0af40"}, + {file = "psycopg2-2.9.13.tar.gz", hash = "sha256:d36784fc2dae69523ba4b79c7d1d1b4d6e83e87836874f111262f4db940b16a6"}, +] + +[[package]] +name = "pycparser" +version = "3.0" +description = "C parser in Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "pycparser-3.0-py3-none-any.whl", hash = "sha256:b727414169a36b7d524c1c3e31839a521725078d7b2ff038656844266160a992"}, + {file = "pycparser-3.0.tar.gz", hash = "sha256:600f49d217304a5902ac3c37e1281c9fe94e4d0489de643a9504c5cdfdfc6b29"}, ] [[package]] @@ -1917,6 +2259,23 @@ files = [ [package.extras] windows-terminal = ["colorama (>=0.4.6)"] +[[package]] +name = "pyjwt" +version = "2.15.1" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyjwt-2.15.1-py3-none-any.whl", hash = "sha256:42d59d631f7768a1028a64c7ff581a9bf7519804daf91fc5b6c56e30eec5e193"}, + {file = "pyjwt-2.15.1.tar.gz", hash = "sha256:4f259e80cdfb6b3fc18a7de51fd1ef9ec79652f25019bae68975ca2468a34df8"}, +] + +[package.dependencies] +typing_extensions = {version = ">=4.0", markers = "python_version < \"3.11\""} + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] + [[package]] name = "pylint" version = "3.3.9" @@ -2034,6 +2393,25 @@ files = [ {file = "pyodbc-5.3.0.tar.gz", hash = "sha256:2fe0e063d8fb66efd0ac6dc39236c4de1a45f17c33eaded0d553d21c199f4d05"}, ] +[[package]] +name = "pyopenssl" +version = "26.4.0" +description = "Python wrapper module around the OpenSSL library" +optional = false +python-versions = ">=3.9" +files = [ + {file = "pyopenssl-26.4.0-py3-none-any.whl", hash = "sha256:f0eb0cb2d581d3ad2b9c489468485e7f2ab6727d08401bcf9d824c3caddf3c1c"}, + {file = "pyopenssl-26.4.0.tar.gz", hash = "sha256:28dfcce0162b9211413e26dfbfdf1d24317fbeba18fc93c12400a1856b2a0bc7"}, +] + +[package.dependencies] +cryptography = ">=49.0.0,<51" +typing-extensions = {version = ">=4.9", markers = "python_version < \"3.13\""} + +[package.extras] +docs = ["sphinx (!=5.2.0,!=5.2.0.post0,!=7.2.5)", "sphinx_rtd_theme"] +test = ["pretend", "pytest (>=3.0.1)", "pytest-rerunfailures"] + [[package]] name = "pytest" version = "9.1.1" @@ -2111,6 +2489,17 @@ files = [ {file = "pytz-2026.2.tar.gz", hash = "sha256:0e60b47b29f21574376f218fe21abc009894a2321ea16c6754f3cad6eb7cdd6a"}, ] +[[package]] +name = "pywin32-ctypes" +version = "0.2.3" +description = "A (partial) reimplementation of pywin32 using ctypes/cffi" +optional = false +python-versions = ">=3.6" +files = [ + {file = "pywin32-ctypes-0.2.3.tar.gz", hash = "sha256:d162dc04946d704503b2edc4d55f3dba5c1d539ead017afa00142c38b9885755"}, + {file = "pywin32_ctypes-0.2.3-py3-none-any.whl", hash = "sha256:8a1513379d709975552d202d942d9837758905c8d01eb82b8bcc30918929e7b8"}, +] + [[package]] name = "pyyaml" version = "6.0.3" @@ -2408,6 +2797,23 @@ sphinx = ["sphinx (>=4.0,<6.0)"] testing = ["coverage-conditional-plugin (>=0.5)", "coverage[toml] (>=6.0)", "pytest (>=6.0)", "pytest-cov (>=3.0)", "pytest-mock (>=3.7)", "pytest-randomly (>=3.0)", "pytest-sugar (>=0.9.5)"] toml = ["tomli (>=2.0,<3.0)"] +[[package]] +name = "s3transfer" +version = "0.19.2" +description = "An Amazon S3 Transfer Manager" +optional = false +python-versions = ">=3.10" +files = [ + {file = "s3transfer-0.19.2-py3-none-any.whl", hash = "sha256:d8168eccca828cbb2cd573675333f3bddd254313a9c42494b84c76b539e8ba25"}, + {file = "s3transfer-0.19.2.tar.gz", hash = "sha256:ba0309fd86be3c27dbf78cdd813c13c5e1df16e5874b99d2535ebbdfb9892993"}, +] + +[package.dependencies] +botocore = ">=1.37.4,<2.0a.0" + +[package.extras] +crt = ["botocore[crt] (>=1.37.4,<2.0a.0)"] + [[package]] name = "scramp" version = "1.4.12" @@ -2422,6 +2828,21 @@ files = [ [package.dependencies] asn1crypto = ">=1.5.1" +[[package]] +name = "secretstorage" +version = "3.5.0" +description = "Python bindings to FreeDesktop.org Secret Service API" +optional = false +python-versions = ">=3.10" +files = [ + {file = "secretstorage-3.5.0-py3-none-any.whl", hash = "sha256:0ce65888c0725fcb2c5bc0fdb8e5438eece02c523557ea40ce0703c266248137"}, + {file = "secretstorage-3.5.0.tar.gz", hash = "sha256:f04b8e4689cbce351744d5537bf6b1329c6fc68f91fa666f60a380edddcd11be"}, +] + +[package.dependencies] +cryptography = ">=2.0" +jeepney = ">=0.6" + [[package]] name = "shellingham" version = "1.5.4" @@ -2474,6 +2895,97 @@ files = [ {file = "snowballstemmer-3.1.1.tar.gz", hash = "sha256:e07bbc54a0d798fe6010a12398422e62a8bfbba95c394fd0956ef58cb4d3e260"}, ] +[[package]] +name = "snowflake-connector-python" +version = "4.6.0" +description = "Snowflake Connector for Python" +optional = false +python-versions = ">=3.10" +files = [ + {file = "snowflake_connector_python-4.6.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:3ff98c3213674c5ed18ba6bb9288c4e88e790150f350824434d49a23d15c0fc3"}, + {file = "snowflake_connector_python-4.6.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:04ea8906ac06bdf98ab265f7870b532f32dd2b0f6b3b06a542b6e25a43e01665"}, + {file = "snowflake_connector_python-4.6.0-cp310-cp310-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:324b15278ee84ea6f0af7fef5e916778c23c4569b2c8ba7fdc90d288478772b9"}, + {file = "snowflake_connector_python-4.6.0-cp310-cp310-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:fe9005d226b234bf190409e5d7e8db9f7daba271880de9105f5173a6858b8e6b"}, + {file = "snowflake_connector_python-4.6.0-cp310-cp310-win_amd64.whl", hash = "sha256:8edc8bbcbaaa25a08d43f943fe45f00dc465684ef243859b0f3f7498d800f1ce"}, + {file = "snowflake_connector_python-4.6.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:03b0a232d8d0a1c78eb0d4e9f8a422a1553b2f69ef1387d50a3223bb1829a249"}, + {file = "snowflake_connector_python-4.6.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:531dcb07eee8405e5d8a9f4e7f8c1ca7916e3afbb4ffb3dd2c9a12ec5bd0e46a"}, + {file = "snowflake_connector_python-4.6.0-cp311-cp311-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:c3124fd4a5dc702173ccd73d821ceba1442134d5f347b4c8d1ecb76489f44671"}, + {file = "snowflake_connector_python-4.6.0-cp311-cp311-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:7ab64f46b18d77d1e6c159a29cd86eeff0be9ff01a9904fa873a3c29d20063d1"}, + {file = "snowflake_connector_python-4.6.0-cp311-cp311-win_amd64.whl", hash = "sha256:18cc5402695b8e958503d6d7ab96403db90c481b63c31520305876ef3cb797e9"}, + {file = "snowflake_connector_python-4.6.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:a7701b702dbeb348769c5d1248231e18544c4ff1fb4118ad73d48e8f801cfb6e"}, + {file = "snowflake_connector_python-4.6.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:00abbcfe958f60da18297191f3499b1e61802e64622521a2e8da1c059c14e1c0"}, + {file = "snowflake_connector_python-4.6.0-cp312-cp312-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:72aaee21a70e00fbe4dadcc60b9b1012b6411dddc90f94804d5efe5706fb9621"}, + {file = "snowflake_connector_python-4.6.0-cp312-cp312-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:6d3f6120edeb0d6edd208831d006cc3e769ec51bc346727f22d7aeaecbf20f77"}, + {file = "snowflake_connector_python-4.6.0-cp312-cp312-win_amd64.whl", hash = "sha256:f15e2493a316ce79ab3d7fb16add10252bb2401723e5cfbc7a2ebc44d89a7b2b"}, + {file = "snowflake_connector_python-4.6.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:e0ca5a035b1afa690fb36a767ba59c8db85ef6295b88c2bbc2040449e99992ad"}, + {file = "snowflake_connector_python-4.6.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:1894504c69a76ac4a205d01fbb3e18c6a6e974e6ad26dad263edd06343bea501"}, + {file = "snowflake_connector_python-4.6.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:ed40d1e9d867253596860b9d5240280489ff4692b7a3fa21e2d45d63b4b61d36"}, + {file = "snowflake_connector_python-4.6.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:1c8476781cfef961fc5f6f75a5238e668d3e0ca5ebf1d055661b2fcf2831c254"}, + {file = "snowflake_connector_python-4.6.0-cp313-cp313-win_amd64.whl", hash = "sha256:e8ccbf8b5e12177a86bd3ab8292cc5a99e9ac97d7645ef4a3ed0f767b4ec6594"}, + {file = "snowflake_connector_python-4.6.0-cp314-cp314-macosx_14_0_arm64.whl", hash = "sha256:1fe93d88278a0b7e0efde6140890bc298a49fbf1e04968a35aa22c801131cced"}, + {file = "snowflake_connector_python-4.6.0-cp314-cp314-macosx_14_0_x86_64.whl", hash = "sha256:0829d57467bf1bb5af411f6e7723058cb2218fb7df07cf15d912e3b1a2c126eb"}, + {file = "snowflake_connector_python-4.6.0-cp314-cp314-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:676162cd45df744aa966483960d34bf204cdcae87cecad77fba970f1c2fd570d"}, + {file = "snowflake_connector_python-4.6.0-cp314-cp314-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:eab420406a38ebc059100bb1faa55d7d6306bb224cefadb739ec3cafeff65384"}, + {file = "snowflake_connector_python-4.6.0-cp314-cp314-win_amd64.whl", hash = "sha256:9dd8689123a7e7b873db0846f2d92745a02062b16665d20634fbaf34a9c88e7a"}, + {file = "snowflake_connector_python-4.6.0.tar.gz", hash = "sha256:06e2dba02703da6fd60e07bb0574506f810a85e5831d3461247753ecce4b8335"}, +] + +[package.dependencies] +asn1crypto = ">0.24.0,<2.0.0" +boto3 = ">=1.24" +botocore = ">=1.24" +certifi = ">=2024.7.4" +charset_normalizer = ">=2,<4" +cryptography = ">=46.0.5" +filelock = ">=3.5,<4" +idna = ">=3.7,<4" +keyring = {version = ">=23.1.0,<26.0.0", optional = true, markers = "extra == \"secure-local-storage\""} +packaging = "*" +platformdirs = ">=2.6.0,<5.0.0" +pyjwt = ">=2.10.1,<3.0.0" +pyOpenSSL = ">=24.0.0" +pytz = "*" +requests = ">=2.32.4,<3.0.0" +sortedcontainers = ">=2.4.0" +tomlkit = "*" +typing_extensions = ">=4.3,<5" + +[package.extras] +boto = ["boto3 (>=1.24)", "botocore (>=1.24)"] +development = ["Cython", "coverage", "mitmproxy (>=12.0.0)", "more-itertools", "numpy (<=2.4.3)", "pendulum (!=2.1.1)", "pexpect", "pytest (<7.5.0)", "pytest-asyncio", "pytest-cov", "pytest-rerunfailures (<16.0)", "pytest-timeout", "pytest-xdist", "pytzdata", "responses"] +pandas = ["pandas (>=1.0.0,<3.0.0)", "pandas (>=2.1.2,<3.0.0)", "pyarrow (>=14.0.1)"] +secure-local-storage = ["keyring (>=23.1.0,<26.0.0)"] + +[[package]] +name = "snowflake-sqlalchemy" +version = "1.11.1" +description = "Snowflake SQLAlchemy Dialect" +optional = false +python-versions = ">=3.8" +files = [ + {file = "snowflake_sqlalchemy-1.11.1-py3-none-any.whl", hash = "sha256:b64ab947469bf25b3df3f88c5b92f17a938c9d47b5b09035551614f3ff1f44d9"}, + {file = "snowflake_sqlalchemy-1.11.1.tar.gz", hash = "sha256:942b203de0404b79550c4098ca851aee2b0b22f695426b073e58d71772e38d51"}, +] + +[package.dependencies] +snowflake-connector-python = "<5.0.0" +sqlalchemy = ">=1.4.19" + +[package.extras] +development = ["alembic", "numpy", "pre-commit", "pytest", "pytest-cov", "pytest-rerunfailures", "pytest-timeout", "pytest-xdist", "pytz", "setuptools", "syrupy"] +pandas = ["snowflake-connector-python[pandas]"] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + [[package]] name = "sphinx" version = "5.3.0" @@ -3168,10 +3680,29 @@ files = [ [package.extras] dev = ["pytest", "setuptools"] +[[package]] +name = "zipp" +version = "4.1.2" +description = "Backport of pathlib-compatible object wrapper for zip files" +optional = false +python-versions = ">=3.10" +files = [ + {file = "zipp-4.1.2-py3-none-any.whl", hash = "sha256:b30cb6cd253122169c47de245ca24a86bac2a5d3802ba15bf7ba6414eb9e144d"}, + {file = "zipp-4.1.2.tar.gz", hash = "sha256:cf474b90b07ea932637e17c6b9479de2b2882900b0d311bdaf4f5c000a15a97f"}, +] + +[package.extras] +check = ["pytest-checkdocs (>=2.14)", "pytest-ruff (>=0.2.1)"] +cover = ["pytest-cov"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +enabler = ["pytest-enabler (>=3.4)"] +test = ["big-O", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more_itertools", "pytest (>=6,!=8.1.*)", "pytest-ignore-flaky"] +type = ["pytest-mypy (>=1.0.1)"] + [extras] docs = ["sphinx-rtd-theme", "sphinxcontrib-napoleon"] [metadata] lock-version = "2.0" python-versions = ">=3.10,<3.14" -content-hash = "32e39aeac70fde68398a8a5d467c29499cd74d177beda122a19f4641689f71c3" +content-hash = "fb28fcff042aa2642bf5ccdf667f6f58c6583fbd5d3d6ec545c9725b09843d87" diff --git a/pyproject.toml b/pyproject.toml index 09096059..2b201cd7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "datafaker" -version = "0.5.3" +version = "0.5.4" description = "Generates fake SQL data" authors = ["Tim Band <3266052+tim-band@users.noreply.github.com>"] license = "MIT" @@ -17,7 +17,7 @@ homepage = "https://github.com/SAFEHR-data/datafaker" [tool.poetry.dependencies] python = ">=3.10,<3.14" pydantic = {extras = ["dotenv"], version = "^1.10.2"} -psycopg2-binary = "^2.9.5" +psycopg2 = "^2.9.13" pyodbc = "^5.0.0" aioodbc = "^0.5.0" sqlalchemy-utils = "^0.41.2" @@ -40,6 +40,8 @@ fastparquet = "^2024.11.0" duckdb-sqlalchemy = "^1.5.2.2" ty = "^0.0.43" colorama = "^0.4.6" +snowflake-sqlalchemy = "^1.11.1" +snowflake-connector-python = {version = "4.6.0", extras = ["secure-local-storage"]} [tool.poetry.group.dev.dependencies] isort = "^5.10.1" diff --git a/tests/examples/date.sql b/tests/examples/date.sql index a0155a9d..2f531718 100644 --- a/tests/examples/date.sql +++ b/tests/examples/date.sql @@ -8,31 +8,47 @@ CREATE TABLE public.person ( id INTEGER NOT NULL, name TEXT NOT NULL, timestamp_of_birth TIMESTAMP WITH TIME ZONE NOT NULL, - date_of_birth DATE NOT NULL + date_of_birth DATE NOT NULL, + parent_of INTEGER ); ALTER TABLE ONLY public.person ADD CONSTRAINT person_pkey PRIMARY KEY (id); +ALTER TABLE ONLY public.person + ADD CONSTRAINT parent_of_fkey FOREIGN KEY (parent_of) REFERENCES public.person(id); ALTER TABLE public.person OWNER TO postgres; -INSERT INTO public.person VALUES (1, 'Bobby', '1991-01-08 11:12:13+00:00', '1991-01-08'); -INSERT INTO public.person VALUES (2, 'Mary', '1989-03-04 20:19:18+00:00', '1989-03-04'); +INSERT INTO public.person VALUES (1, 'Bobby', '1991-01-08 11:12:13+00:00', '1991-01-08', NULL); +INSERT INTO public.person VALUES (2, 'Mary', '1989-03-04 20:19:18+00:00', '1989-03-04', NULL); +INSERT INTO public.person VALUES (3, 'Elizabeth', '1951-10-14 00:12:12+00:00', '1951-10-14', 2); CREATE TABLE public.happening ( id INTEGER NOT NULL, name TEXT NOT NULL, person_id INTEGER NOT NULL, at_time TIMESTAMP WITH TIME ZONE NOT NULL, - at_date DATE NOT NULL + at_date DATE NOT NULL, + other_person_id INTEGER, + comment TEXT, + previous_happening_id INTEGER ); ALTER TABLE ONLY public.happening ADD CONSTRAINT happening_pkey PRIMARY KEY (id); ALTER TABLE ONLY public.happening ADD CONSTRAINT person_id_fkey FOREIGN KEY (person_id) REFERENCES public.person(id); +ALTER TABLE ONLY public.happening + ADD CONSTRAINT other_person_id_fkey FOREIGN KEY (other_person_id) REFERENCES public.person(id); +ALTER TABLE ONLY public.happening + ADD CONSTRAINT previous_happening_id_fkey FOREIGN KEY (previous_happening_id) REFERENCES public.happening(id); ALTER TABLE public.happening OWNER TO postgres; -INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20'); -INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20'); -INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23'); -INSERT INTO public.happening VALUES (4, 'steppen in gum', 2, '2003-11-03 04:15:26+00:00', '2003-11-03'); +INSERT INTO public.happening VALUES (1, 'stepped on a tack', 1, '1997-04-20 04:05:06+00:00', '1997-04-20', NULL, NULL, NULL); +INSERT INTO public.happening VALUES (2, 'had a dream', 2, '1997-04-20 04:04:16+00:00', '1997-04-20', NULL, NULL, 1); +INSERT INTO public.happening VALUES (3, 'kicked a can', 1, '2001-12-23 07:05:06+00:00', '2001-12-23', NULL, NULL, 1); +INSERT INTO public.happening VALUES (4, 'shared a cake', 2, '2003-11-03 04:15:26+00:00', '2003-11-03', 1, NULL, NULL); + +CREATE VIEW public.happening_view (id, name, person_id) + AS SELECT id, name, person_id FROM public.happening; + +ALTER VIEW public.happening_view OWNER TO postgres; diff --git a/tests/test_evaluators_proposal_ranking.py b/tests/test_evaluators_proposal_ranking.py index dbf0182a..64c033f5 100644 --- a/tests/test_evaluators_proposal_ranking.py +++ b/tests/test_evaluators_proposal_ranking.py @@ -1,6 +1,8 @@ """Unit tests for datafaker.evaluators.proposal_ranking.""" from typing import Any +from sqlalchemy import Column, MetaData, Table, Text + from datafaker.evaluators.column_evaluator import EvaluationProfile, ProposalEvaluation from datafaker.evaluators.proposal_ranking import ( ProposalRanking, @@ -227,7 +229,8 @@ def test_keyword_boost_can_flip_the_recommendation(self) -> None: def test_choice_proposer_penalized_by_resampling_on_unique_column(self) -> None: """ChoiceProposer's score is discounted by real_uniqueness x copy_fraction.""" - choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + table = Table("t", MetaData(), Column("c", Text())) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) results = [ make_result(overall_score=0.0, copy_fraction=1.0, proposer=choice_proposer), make_result(overall_score=0.5, copy_fraction=0.0), @@ -307,7 +310,8 @@ def test_combined_score_is_clamped_to_one(self) -> None: def test_weak_recommendation_warning_when_best_fit_was_suppressed(self) -> None: """A heavily-discounted best-fit candidate triggers the fallback warning.""" - choice_proposer = UniformChoiceProposer("t", "c", ["a", "b"], [1, 1]) + table = Table("t", MetaData(), Column("c", Text())) + choice_proposer = UniformChoiceProposer(table.c.c, ["a", "b"], [1, 1]) results = [ # Fits almost perfectly but is a resampler on a near-unique column: # gets discounted to (near) zero. diff --git a/tests/test_functional_mssql.py b/tests/test_functional_mssql.py index dbd6415f..89191868 100644 --- a/tests/test_functional_mssql.py +++ b/tests/test_functional_mssql.py @@ -17,6 +17,7 @@ from tempfile import mkstemp import yaml +from sqlalchemy import Column, MetaData, Table, Text from sqlalchemy import create_engine as sa_create_engine from sqlalchemy import text from sqlalchemy.dialects import mssql as mssql_dialect # noqa: PLC0415 @@ -52,7 +53,9 @@ def setUp(self) -> None: (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as fh: fh.write( - make_tables_file(self.dsn, self.schema_name, engine=self.sync_engine) + make_tables_file( + self.dsn, self.schema_name, False, engine=self.sync_engine + ) ) def tearDown(self) -> None: @@ -119,9 +122,9 @@ def test_dialect_rand(self) -> None: """ChoiceProposer compiles its query with NEWID() not RANDOM() for mssql.""" dialect = mssql_dialect.dialect() + table = Table("manufacturer", MetaData(), Column("name", Text())) proposer = ZipfChoiceProposer( - table_name="manufacturer", - column_name="name", + column=table.c.name, values=["Blender", "Gibbs"], counts=[5, 5], sample_count=2, diff --git a/tests/test_generators_dialect.py b/tests/test_generators_dialect.py index ec5e1744..99261c12 100644 --- a/tests/test_generators_dialect.py +++ b/tests/test_generators_dialect.py @@ -4,14 +4,19 @@ import unittest.mock from unittest.mock import MagicMock -from sqlalchemy import Column, Integer, MetaData, Select, Table, literal_column +from sqlalchemy import Column, Integer, MetaData, Select, Table, Text, literal_column from sqlalchemy.dialects import mssql, postgresql from sqlalchemy.types import DateTime from datafaker.dialects import SecondsDifference from datafaker.interactive.generators import get_aggregate_query from datafaker.interactive.missingness import MissingnessType -from datafaker.proposers.base import Buckets, PredefinedProposer, Proposer +from datafaker.proposers.base import ( + Buckets, + PredefinedProposer, + Proposer, + RelatedColumn, +) from datafaker.proposers.choice import ChoiceProposerFactory, ZipfChoiceProposer from datafaker.proposers.continuous import ( ContinuousLogDistributionProposerFactory, @@ -141,9 +146,13 @@ class TestChoiceGeneratorStoredQuery(DatafakerTestCase): """ChoiceGenerator._query is compiled to dialect-correct SQL at construction time.""" def _make_gen(self, dialect, sample_count=None, suppress_count=0): + table = Table( + "patient", + MetaData(), + Column("gender", Text()), + ) return ZipfChoiceProposer( - table_name="patient", - column_name="gender", + column=table.c.gender, values=["M", "F"], counts=[70, 30], sample_count=sample_count, @@ -592,7 +601,7 @@ def test_date_after_proposer_stddev_clause_mssql(self) -> None: sd=1.0, mean=86400.0, column=tbl.c.end_date, - anchor=tbl.c.start_date, + anchor_related=RelatedColumn(tbl.c.start_date, None), engine=engine, ) clauses = proposer.select_aggregate_clauses() @@ -620,7 +629,7 @@ def test_date_after_proposer_stddev_clause_postgresql(self) -> None: sd=1.0, mean=86400.0, column=tbl.c.end_date, - anchor=tbl.c.start_date, + anchor_related=RelatedColumn(tbl.c.start_date, None), engine=engine, ) clauses = proposer.select_aggregate_clauses() diff --git a/tests/test_interactive_copy_generators.py b/tests/test_interactive_copy_generators.py new file mode 100644 index 00000000..7cda8697 --- /dev/null +++ b/tests/test_interactive_copy_generators.py @@ -0,0 +1,262 @@ +""" Tests for the configure-generators command. """ +import re +from collections.abc import MutableMapping +from typing import Any, Iterable + +from sqlalchemy import select +from sqlalchemy.orm import aliased + +from datafaker.interactive.base import DbCmd +from tests.test_interactive_generators import MockGeneratorCmd +from tests.utils import GeneratesDBTestCase, MsSqlTestDb + + +class ConfigureCopyGeneratorsWithDateTests(GeneratesDBTestCase): + """Test `configure-generators` with the `instrument.sql` database.""" + + dump_file_path = "date.sql" + database_name = "date_tables" + schema_name = "public" + use_temporary_cwd = True + + def _get_cmd(self, config: MutableMapping[str, Any]) -> MockGeneratorCmd: + """Get the command we are using for this test case.""" + return MockGeneratorCmd( + DbCmd.Settings(self.dsn, self.schema_name, config, self.metadata, None) + ) + + def test_same_table_copy(self) -> None: + """Test that the same-table copy is proposed and compared.""" + config = { + "tables": { + "happening": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = "generic.anchored_provider.copy [from name]" + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.reset() + gc.do_compare(str(prop[0])) + self.assertEqual(gc.messages[0][0], gc.NOT_PRIVATE_TEXT) + self.assertEqual(gc.messages[1][0], gc.REQUIRES_NO_SOURCE_DATA_TEXT) + + def test_cross_table_copy(self) -> None: + """Test that the cross-table copy is proposed and compared.""" + config = { + "tables": { + "person": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + provider_name2 = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via other_person_id)]" + ) + self.assertIn(provider_name2, proposals.keys()) + prop2 = proposals[provider_name2] + gc.reset() + gc.do_compare(f"{prop[0]} {prop2[0]}") + self.assertEqual(gc.messages[0][0], gc.NOT_PRIVATE_TEXT) + self.assertEqual(gc.messages[1][0], gc.REQUIRES_NO_SOURCE_DATA_TEXT) + + def test_cross_table_copy_actually_copies(self) -> None: + """Test that the cross-table copy is copies.""" + config = { + "tables": { + "person": { + "columns": { + "name": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + # set up our copy proposer + gc.do_next("happening.name") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.do_set(str(prop[0])) + gc.do_next("happening.comment") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from name of table person (via other_person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + prop = proposals[provider_name] + gc.do_set(str(prop[0])) + gc.do_quit("") + self.generate_data(gc.config, num_passes=50) + assert self.dst_engine is not None + with self.dst_engine.connect() as conn: + pert = self.dst_metadata.tables["person"] + other_pert = aliased(pert) + hapt = self.dst_metadata.tables["happening"] + query = ( + select( + hapt.columns["name"].label("hn"), + hapt.columns["comment"].label("hc"), + pert.columns["name"].label("pn"), + other_pert.columns["name"].label("opn"), + ) + .select_from(hapt) + .join( + pert, + onclause=pert.columns["id"] == hapt.columns["person_id"], + ) + .join( + other_pert, + onclause=other_pert.columns["id"] + == hapt.columns["other_person_id"], + isouter=True, + ) + ) + dst_result = conn.execute(query).fetchall() + saw_other = False + for row in dst_result: + if row.opn: + saw_other = True + self.assertEqual(row.hc, row.opn) + self.assertEqual(row.hn, row.pn) + self.assertTrue(saw_other) + + def test_cross_table_fk_copy(self) -> None: + """Test that the cross-table foreign key copy is proposed.""" + config = { + "tables": { + "person": { + "columns": { + "parent_of": { + "roles": ["source"], + } + } + } + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.copy_fk" + " [from parent_of of table person (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + + COPY_FROM_RE = re.compile(r"generic.anchored_provider.copy \[from ([^\]]+)\]") + + def get_same_table_copy_proposals(self, proposals: Iterable[str]) -> set[str]: + """Return a set of columns proposed to copy from.""" + return { + p.group(1) + for p in (self.COPY_FROM_RE.match(prop) for prop in proposals) + if p is not None + } + + def test_different_target_fk_copy_is_not_proposed(self) -> None: + """Test a foreign key copy with a different target table is not proposed.""" + config = { + "tables": { + "happening": { + "columns": { + "previous_happening_id": { + "roles": ["source"], + }, + "person_id": { + "roles": ["source"], + }, + } + } + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = "generic.anchored_provider.copy [from person_id]" + self.assertIn(provider_name, proposals.keys()) + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertIn("person_id", all_copies) + self.assertNotIn("previous_happening_id", all_copies) + + def test_nullable_copy_nonnullable_is_not_proposed(self) -> None: + """Test a copy from a nullable column to a nonnullable column is not proposed.""" + config = { + "tables": { + "happening": { + "columns": { + "other_person_id": { + "roles": ["source"], + }, + "person_id": { + "roles": ["source"], + }, + } + } + } + } + with self._get_cmd(config) as gc: + # person -> other_person is allowed + gc.do_next("happening.other_person_id") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertIn("person_id", all_copies) + # other_person -> person is allowed + gc.do_next("happening.person_id") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + all_copies = self.get_same_table_copy_proposals(proposals.keys()) + self.assertNotIn("other_person_id", all_copies) + + +# Note that this test won't work with DuckDB because it needs foreign keys to work +class ConfigureCopyGeneratorsWithDateMsSqlTests(ConfigureCopyGeneratorsWithDateTests): + """Test `configure-generators` with `instrument.sql` with MS SQL.""" + + database_type = MsSqlTestDb + schema_name = None diff --git a/tests/test_interactive_date_generators.py b/tests/test_interactive_date_generators.py index 5dd18f02..a5f47b2a 100644 --- a/tests/test_interactive_date_generators.py +++ b/tests/test_interactive_date_generators.py @@ -71,7 +71,7 @@ def test_intervals_end_to_end(self) -> None: gc.do_next(f"{table}.{column}") gc.do_unmerge("visit_start") gc.reset() - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date [anchored to visit_start]" @@ -167,11 +167,12 @@ def test_cross_table_interval_end_to_end(self) -> None: # set up our interval proposer gc.do_next(f"{table}.{column}") gc.reset() - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via manufacturer_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] @@ -221,7 +222,7 @@ class ConfigureGeneratorsWithInstrumentsMsSqlTests( class ConfigureGeneratorsWithDateTests(GeneratesDBTestCase): - """Test `configure-generators` with the `instrument.sql` database.""" + """Test `configure-generators` with the `date.sql` database.""" dump_file_path = "date.sql" database_name = "date_tables" @@ -281,13 +282,15 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] provider_name2 = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor2} of table {atable}]" + f" [anchored to {anchor2} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name2, proposals.keys()) prop2 = proposals[provider_name2] @@ -302,13 +305,15 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: proposals = gc.get_proposals() provider_name = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor} of table {atable}]" + f" [anchored to {anchor} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name, proposals.keys()) prop = proposals[provider_name] provider_name2 = ( "generic.anchored_provider.normal_date_fk" - f" [anchored to {anchor2} of table {atable}]" + f" [anchored to {anchor2} of table {atable}" + " (via person_id)]" ) self.assertIn(provider_name2, proposals.keys()) prop2 = proposals[provider_name2] @@ -318,10 +323,35 @@ def test_cross_table_interval_timestamp_vs_date(self) -> None: self.assertEqual(gc.messages[1][0], gc.REQUIRES_SOURCE_DATA_TEXT) self.assertEqual(gc.messages[2][0], gc.PROVIDING_VALUES_TEXT) + def test_regression_date_start_crash(self) -> None: + """Test for regression where ``start`` role on a date leads to crash.""" + config = { + "tables": { + "person": { + "columns": { + "date_of_birth": { + "roles": ["start"], + }, + }, + }, + } + } + with self._get_cmd(config) as gc: + gc.do_next("happening.at_date") + gc.reset() + gc.do_propose("all") + proposals = gc.get_proposals() + provider_name = ( + "generic.anchored_provider.normal_date_fk" + " [anchored to date_of_birth of table person" + " (via person_id)]" + ) + self.assertIn(provider_name, proposals.keys()) + # Note that this test won't work with DuckDB because it needs foreign keys to work class ConfigureGeneratorsWithDateMsSqlTests(ConfigureGeneratorsWithDateTests): - """Test `configure-generators` with `instrument.sql` with MS SQL.""" + """Test `configure-generators` with `date.sql` with MS SQL.""" database_type = MsSqlTestDb schema_name = None diff --git a/tests/test_interactive_generators.py b/tests/test_interactive_generators.py index 46b20c30..de84689a 100644 --- a/tests/test_interactive_generators.py +++ b/tests/test_interactive_generators.py @@ -63,7 +63,7 @@ def test_null_configuration(self) -> None: with self._get_cmd(config) as gc: table = "model" gc.do_next(f"{table}.name") - gc.do_propose("") + gc.do_propose("all") gc.do_compare("") gc.do_set("1") gc.do_quit("") @@ -79,7 +79,7 @@ def test_null_table_configuration(self) -> None: with self._get_cmd(config) as gc: table = "model" gc.do_next(f"{table}.name") - gc.do_propose("") + gc.do_propose("all") gc.do_set("1") gc.do_quit("") self.assertEqual(len(gc.config["tables"][table]["row_generators"]), 1) @@ -151,7 +151,7 @@ def test_set_generator_distribution(self) -> None: column = "frequency" generator = "dist_gen.normal" gc.do_next(f"{table}.{column}") - gc.do_propose("") + gc.do_propose("all") proposals = gc.get_proposals() gc.do_set(str(proposals[generator][0])) gc.do_quit("") @@ -240,9 +240,9 @@ def test_set_generator_choice(self) -> None: gc.config["src-stats"][0]["query"], ( "SELECT _counted.value \n" - f'FROM (SELECT "{column}" AS value, count("{column}") AS count \n' + f"FROM (SELECT {table}.{column} AS value, count({table}.{column}) AS count \n" f"FROM {table} \n" - f'WHERE "{column}" IS NOT NULL GROUP BY "{column}") ' + f"WHERE {table}.{column} IS NOT NULL GROUP BY {table}.{column}) " "AS _counted ORDER BY _counted.count DESC" ), ) diff --git a/tests/test_interactive_missingness.py b/tests/test_interactive_missingness.py index c1da7c54..747f9e0f 100644 --- a/tests/test_interactive_missingness.py +++ b/tests/test_interactive_missingness.py @@ -1,7 +1,8 @@ """ Tests for the configure-missingness command. """ import random from collections.abc import MutableMapping -from typing import Any +from pathlib import Path +from typing import Any, Iterable from sqlalchemy import select @@ -10,6 +11,15 @@ from tests.utils import GeneratesDBTestCase, RequiresDBTestCase, TestDbCmdMixin +def count_nonnull(it: Iterable): + """Count the number of non-null values in an iterable.""" + n = 0 + for v in it: + if v is not None: + n += 1 + return n + + class MockMissingnessCmd(MissingnessCmd, TestDbCmdMixin): """MissingnessCmd but mocked""" @@ -107,3 +117,39 @@ def test_create_with_missingness(self) -> None: patterns.add(p + b) # all pattern possibilities should be present self.assertSetEqual(patterns, {0, 1, 2, 3}) + + +class ConfigureMissingnessEndToEndTests(ConfigureMissingnessTestsWithGeneration): + """Test `configure-missing` end-to-end.""" + + use_temporary_cwd = True + copy_files = ["row_generators.py", "story_generators.py"] + copy_from_directory = Path("examples") + + def test_missingness_end_to_end(self) -> None: + """Test that if an interval end is applicable it gets proposed and works.""" + table = "signature_model" + with self._get_cmd({}) as mc: + mc.do_next(table) + mc.do_sampled("") + mc.reset() + mc.do_quit("") + self.generate_data(mc.config, num_passes=100) + self.assertIs( + type( + mc.config["tables"][table]["missingness_generators"][0]["columns"][ + 0 + ] + ), + str, + ) + assert self.dst_engine is not None + with self.dst_engine.connect() as conn: + stmt = select(self.metadata.tables[table]) + rows = conn.execute(stmt).mappings().fetchall() + name_count = count_nonnull(r.name for r in rows) + player_id_count = count_nonnull(r.player_id for r in rows) + based_on_count = count_nonnull(r.based_on for r in rows) + self.assertEqual(name_count, 100) + self.assertAlmostEqual(player_id_count, 50, delta=20) + self.assertAlmostEqual(based_on_count, 70, delta=20) diff --git a/tests/test_main.py b/tests/test_main.py index 02878b14..ab9819f4 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -111,7 +111,10 @@ def test_make_tables( ) mock_make_tables_file.assert_called_once_with( - "postgresql://suser:spassword@shost:5432/sdbname", None, None + "postgresql://suser:spassword@shost:5432/sdbname", + None, + views_to_tables=False, + parquet_dir=None, ) mock_path.return_value.write_text.assert_called_once_with( "some text", encoding="utf-8" @@ -176,6 +179,8 @@ def test_make_tables_with_force_enabled( for force_option in ["--force", "-f"]: with self.subTest(f"Using option {force_option}"): + mock_make_tables.reset_mock() + mock_path.reset_mock() result: Result = runner.invoke( app, [ @@ -188,16 +193,14 @@ def test_make_tables_with_force_enabled( mock_make_tables.assert_called_once_with( mock_get_settings.return_value.src_dsn, mock_get_settings.return_value.src_schema, - None, + views_to_tables=False, + parquet_dir=None, ) mock_path.return_value.write_text.assert_called_once_with( mock_tables_output, encoding="utf-8" ) self.assertSuccess(result) - mock_make_tables.reset_mock() - mock_path.reset_mock() - @patch("datafaker.main.Path") @patch("datafaker.settings.get_settings") def test_incorrect_dialect_causes_nice_error_message( diff --git a/tests/test_make.py b/tests/test_make.py index 9327ea6a..5578ad79 100644 --- a/tests/test_make.py +++ b/tests/test_make.py @@ -3,6 +3,7 @@ import tempfile from pathlib import Path from typing import Any +from unittest import SkipTest from unittest.mock import MagicMock, patch import pandas as pd @@ -20,12 +21,93 @@ from sqlalchemy.dialects.mysql.types import INTEGER from sqlalchemy.dialects.postgresql import UUID +from datafaker.db_utils import create_db_engine, get_sync_engine from datafaker.make import ( _get_default_generator, _get_provider_for_column, make_src_stats, + make_tables_file, ) -from tests.utils import DatafakerTestCase, GeneratesDBTestCase, RequiresDBTestCase +from tests.utils import ( + DatafakerTestCase, + GeneratesDBTestCase, + MsSqlTestDb, + PostgresTestDb, + RequiresDBTestCase, +) + + +class TestSerialize(DatafakerTestCase): + """Unit tests for serialize_metadata.""" + + database_type = PostgresTestDb + + @classmethod + def setUpClass(cls) -> None: + super().setUpClass() + skip_msg = cls.database_type.skip() + if skip_msg: + raise SkipTest(skip_msg) + cls.database_type.setup() + + def setUp(self) -> None: + super().setUp() + self.database = self.database_type() + self.database.open() + self.database.run_sql(self.get_abs_example_dir() / "date.sql") + self.engine = create_db_engine(self.database.get_dsn("date_tables"), None) + + def test_metadata_to_dict(self) -> None: + """Test metadata_to_dict.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=False, + parquet_dir=None, + engine=get_sync_engine(self.engine), + ) + orm = yaml.load(orm_yaml, yaml.Loader) + table_names = set(orm["tables"].keys()) + self.assert_subset({"person", "happening"}, table_names) + self.assertNotIn("happening_view", table_names) + happening_columns = orm["tables"]["happening"]["columns"] + happening_column_names = set(happening_columns.keys()) + self.assert_subset( + {"id", "name", "comment", "person_id"}, + happening_column_names, + ) + self.assertEqual(happening_columns["person_id"]["type"], "INTEGER") + self.assertFalse(happening_columns["name"]["primary"]) + self.assertTrue(happening_columns["id"]["primary"]) + + def test_views_to_dict(self) -> None: + """Test views_to_tables.""" + orm_yaml = make_tables_file( + self.database.get_dsn("date_tables"), + schema_name=None, + views_to_tables=True, + parquet_dir=None, + engine=get_sync_engine(self.engine), + ) + orm = yaml.load(orm_yaml, yaml.Loader) + table_names = set(orm["tables"].keys()) + self.assert_subset({"person", "happening", "happening_view"}, table_names) + happening_columns = orm["tables"]["happening_view"]["columns"] + happening_column_names = set(happening_columns.keys()) + self.assert_subset( + {"id", "name", "person_id"}, + happening_column_names, + ) + self.assertEqual(happening_columns["person_id"]["type"], "INTEGER") + self.assertFalse(happening_columns["name"]["primary"]) + # FK and PK constraints are not visible through views + # self.assertTrue(happening_columns["id"]["primary"]) + + +class TestSerializeMsSql(TestSerialize): + """Unit tests for serialize_metadata with an MS SQL database.""" + + database_type = MsSqlTestDb class TestGetDefaultGenerator(DatafakerTestCase): diff --git a/tests/test_utils_mssql.py b/tests/test_utils_mssql.py index 5e27cd8f..44748ed0 100644 --- a/tests/test_utils_mssql.py +++ b/tests/test_utils_mssql.py @@ -107,8 +107,9 @@ def test_duckdb_parquet_dir_sets_search_path(self) -> None: if len(calls[0].args) > 1 else calls[0].kwargs.get("settings", {}) ) - self.assertIn("file_search_path", settings_passed) - self.assertEqual(settings_passed["file_search_path"], f"'{parq_dir}'") + all_settings = "\n".join(settings_passed) + self.assertIn("file_search_path", all_settings) + self.assertIn(f"SET file_search_path TO '{parq_dir}';", all_settings) def test_mssql_dsn_schema_sets_translate_map(self) -> None: """schema_translate_map is set even for an MS-SQL DSN (engine creation, no connect).""" @@ -144,9 +145,11 @@ def test_reflect_called_with_schema(self) -> None: mock_meta_data.return_value = mock_md mock_md.reflect.return_value = None - get_metadata(mock_engine, schema_name="myschema") + get_metadata(mock_engine, schema_name="myschema", views_to_tables=False) - mock_md.reflect.assert_called_once_with(mock_engine, schema="myschema") + mock_md.reflect.assert_called_once_with( + mock_engine, schema="myschema", views=False + ) def test_reflect_called_without_schema_when_none(self) -> None: """get_metadata passes schema=None to reflect when no schema_name is given.""" @@ -158,6 +161,8 @@ def test_reflect_called_without_schema_when_none(self) -> None: mock_meta_data.return_value = mock_md mock_md.reflect.return_value = None - get_metadata(mock_engine) + get_metadata(mock_engine, schema_name=None, views_to_tables=False) - mock_md.reflect.assert_called_once_with(mock_engine, schema=None) + mock_md.reflect.assert_called_once_with( + mock_engine, schema=None, views=False + ) diff --git a/tests/utils.py b/tests/utils.py index 8bdf5057..4e51d9fa 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -284,7 +284,7 @@ class MsSqlTestDb(TestDatabaseBase): _EOL_RE = re.compile(r";$", re.MULTILINE) _TIMESTAMP_RE = re.compile(r"(TIMESTAMP)\s+WITH\s+TIME\s+ZONE", re.IGNORECASE) _ALTER_OWNER_RE = re.compile( - r'ALTER\s+(TABLE|DATABASE)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;' + r'ALTER\s+(TABLE|DATABASE|VIEW)\s+["A-Za-z0-9_.]+\s+OWNER\s+TO\s+[A-Za-z0-9_]+\s*;' ) _ALTER_ONLY_RE = re.compile(r"ALTER\s+TABLE\s+ONLY\s+") _CREATE_INDEX_RE = re.compile( @@ -764,7 +764,7 @@ def setUp(self) -> None: # Generate the `orm.yaml` from the database (self.orm_fd, self.orm_file_path) = mkstemp(".yaml", "orm_", text=True) with os.fdopen(self.orm_fd, "w", encoding="utf-8") as orm_fh: - orm_fh.write(make_tables_file(self.dsn, self.schema_name)) + orm_fh.write(make_tables_file(self.dsn, self.schema_name, False)) # Create a separate empty destination database self.make_destination_database("dst")