diff --git a/pyshex/utils/schema_loader.py b/pyshex/utils/schema_loader.py index 544eb13..4d57669 100644 --- a/pyshex/utils/schema_loader.py +++ b/pyshex/utils/schema_loader.py @@ -1,4 +1,3 @@ -import codecs import os import re from pathlib import PureWindowsPath @@ -23,10 +22,11 @@ def _base_uri(loc: str | None) -> str | None: def load_shex_file(shexfilename: str) -> str: - """ Read a ShEx file or URL, honoring a UTF-8 BOM and otherwise guessing the encoding. + """ Read a ShEx file or URL as UTF-8 (honoring a BOM), guessing the encoding only if that fails. - Same as pyshexc.parser_impl.generate_shexj.load_shex_file, except that it falls back to - UTF-8 when chardet (>= 7) reports no encoding, instead of crashing in bytes.decode(None). + ShExC is UTF-8 by definition, so chardet is a fallback for non-conforming files rather than + the first resort: it samples only the start of the data, and reports ascii with confidence 1.0 + for a large file whose first non-ASCII character comes later (e.g. FHIR R4's fhir.shex). """ if '://' in shexfilename: with request.urlopen(shexfilename) as response: @@ -34,11 +34,15 @@ def load_shex_file(shexfilename: str) -> str: else: with open(shexfilename, 'rb') as inf: data = inf.read() - if data.startswith(codecs.BOM_UTF8): - return data.decode('utf-8-sig') - result = chardet.detect(data) - encoding = result['encoding'] if result['encoding'] and float(result['confidence'] or 0) > 0.9 else 'UTF-8' - return data.decode(encoding) + try: + return data.decode('utf-8-sig') # also strips a UTF-8 BOM if present + except UnicodeDecodeError: + pass + encoding = chardet.detect(data)['encoding'] + try: + return data.decode(encoding or 'latin-1') + except (UnicodeDecodeError, LookupError): + return data.decode('latin-1') # maps every byte, so never fails # ShExJ members holding IRIs that are resolved against the document base. Imports are diff --git a/tests/test_pyshex_utils/test_schema_loader.py b/tests/test_pyshex_utils/test_schema_loader.py index fef35e4..650fa1c 100644 --- a/tests/test_pyshex_utils/test_schema_loader.py +++ b/tests/test_pyshex_utils/test_schema_loader.py @@ -1,8 +1,9 @@ +import codecs import os from rdflib import RDF -from pyshex.utils.schema_loader import SchemaLoader +from pyshex.utils.schema_loader import SchemaLoader, load_shex_file SCHEMAS_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), '..', 'data', 'schemas')) @@ -100,4 +101,34 @@ def test_format_change() -> None: loader.schema_format = 'shex' assert loader.location_rewrite(f"{loc}.shex") == f"{loc}.shex" assert loader.location_rewrite(f"{loc}.shextern") == f"{loc}.shextern" - assert loader.location_rewrite(f"{loc}.jsontern") == f"{loc}.shextern" \ No newline at end of file + assert loader.location_rewrite(f"{loc}.jsontern") == f"{loc}.shextern" + + +# A UTF-8 character after ~300KB of ASCII: chardet only samples the start and reports +# ascii with confidence 1.0 (as it does for FHIR R4's fhir.shex), which used to crash decode() +CAFE_SHEXC = ' { ["caf\u00e9"] }\n' + + +def test_load_shexc_utf8_after_long_ascii_prefix(tmp_path) -> None: + shex_file = tmp_path / 'long.shex' + shex_file.write_bytes(('# padding line for chardet\n' * 12000 + CAFE_SHEXC).encode('utf-8')) + assert load_shex_file(str(shex_file)).endswith(CAFE_SHEXC) + schema = SchemaLoader().load(str(shex_file)) + assert schema.shapes[0].expression.valueExpr.values[0].value == 'caf\u00e9' + + +def test_load_shexc_non_utf8_falls_back(tmp_path) -> None: + """ShExC must be UTF-8, but a Latin-1 file still loads rather than crashing. + Which 8-bit encoding chardet guesses for so short a sample varies, so only the ASCII is checked.""" + shex_file = tmp_path / 'latin1.shex' + shex_file.write_bytes(CAFE_SHEXC.encode('latin-1')) + text = load_shex_file(str(shex_file)) + assert len(text) == len(CAFE_SHEXC) + e_acute = CAFE_SHEXC.index('é') + assert text[:e_acute] + 'é' + text[e_acute + 1:] == CAFE_SHEXC + + +def test_load_shexc_utf8_bom(tmp_path) -> None: + shex_file = tmp_path / 'bom.shex' + shex_file.write_bytes(codecs.BOM_UTF8 + CAFE_SHEXC.encode('utf-8')) + assert load_shex_file(str(shex_file)) == CAFE_SHEXC