diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index 033ee7492c..e4a2888161 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -25,7 +25,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.10", "3.13"] # Lower and higher versions we support + python-version: ["3.10", "3.14"] # Lower and higher versions we support os: [macos-latest, windows-latest, ubuntu-latest] steps: - uses: actions/checkout@v6 @@ -129,7 +129,8 @@ jobs: run: | uv pip install --system datalad-installer if [ ${{ runner.os }} = 'Linux' ]; then - datalad-installer --sudo ok git-annex --method datalad/packages + # TODO: remove the pin once datalad uploads the git-annex 10.20261006 package (its latest-version points to a missing file) + datalad-installer --sudo ok git-annex=10.20260901 --method datalad/packages elif [ ${{ runner.os }} = 'macOS' ]; then datalad-installer --sudo ok git-annex --method brew elif [ ${{ runner.os }} = 'Windows' ]; then diff --git a/conftest.py b/conftest.py index 9b1296bb0b..e4a08c170a 100644 --- a/conftest.py +++ b/conftest.py @@ -48,9 +48,15 @@ def pytest_collection_modifyitems(config, items): Marking them in turn allows the tests to be run by using the pytest -m marker_name option. """ + from spikeinterface.core.core_tools import _is_zarr_write_supported + rootdir = Path(config.rootdir) modules_location = rootdir / "src" / "spikeinterface" for item in items: + # TODO: remove once writing to zarr is supported with zarr>=3 + if item.get_closest_marker("requires_zarr_write") and not _is_zarr_write_supported(): + item.add_marker(pytest.mark.skip(reason="Writing to zarr is not supported yet with zarr>=3")) + if config.getoption("--mp-context") is not None and item.name == "test_global_job_kwargs": item.add_marker(pytest.mark.skip(reason="--mp-context changes the default job kwargs")) diff --git a/pyproject.toml b/pyproject.toml index f9a956a844..32442a7e72 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,10 +20,12 @@ classifiers = [ dependencies = [ - "numpy>=2.0.0", + "numpy>=2.0.0;python_version<'3.14'", + "numpy>=2.3.2;python_version>='3.14'", "threadpoolctl>=3.0.0", "tqdm", - "zarr>=2.18,<3", + "zarr>=2.18,<3;python_version<'3.14'", + "zarr>=3;python_version>='3.14'", # Zarr writing is not yet supported with zarr 3 # for release we need pypi, so this needs to be commented "neo @ git+https://github.com/NeuralEnsemble/python-neo.git", "probeinterface @ git+https://github.com/SpikeInterface/probeinterface.git", @@ -31,7 +33,8 @@ dependencies = [ # "probeinterface>=0.4.0", "packaging", "pydantic", - "numcodecs<0.16.0", # For supporting zarr < 3 + "numcodecs<0.16.0;python_version<'3.14'", # For supporting zarr < 3 + "numcodecs>=0.16.5;python_version>='3.14'", ] [build-system] @@ -364,6 +367,7 @@ markers = [ "widgets", "sortingcomponents", "streaming_extractors: extractors that require streaming such as ross and fsspec", + "requires_zarr_write: tests that write to zarr, skipped with zarr>=3 where writing is not supported yet", ] filterwarnings =[ 'ignore:.*distutils Version classes are deprecated.*:DeprecationWarning', diff --git a/src/spikeinterface/core/core_tools.py b/src/spikeinterface/core/core_tools.py index 519771144c..58d53dca91 100644 --- a/src/spikeinterface/core/core_tools.py +++ b/src/spikeinterface/core/core_tools.py @@ -763,6 +763,27 @@ def measure_memory_allocation(measure_in_process: bool = True) -> float: return memory +def _is_zarr_write_supported() -> bool: + """ + Whether writing to zarr is supported with the installed zarr, which is not yet the case for zarr>=3. + """ + return int(zarr.__version__.split(".")[0]) < 3 + + +def _check_zarr_write_is_supported() -> None: + """ + Raise an informative error when writing to zarr with zarr>=3, which is not supported yet. + + zarr>=3 is what gets installed on Python 3.14, where reading existing zarr folders works but writing does not. + """ + if not _is_zarr_write_supported(): + raise NotImplementedError( + f"Writing to zarr is not supported yet with zarr {zarr.__version__}, which is the version installed " + "on Python 3.14. Use Python 3.13 or lower to save in zarr format, or save in another format such as " + "'binary_folder'." + ) + + def is_path_remote(path: str | Path) -> bool: """ Returns True if the path is a remote path (e.g., s3:// or gcs://). diff --git a/src/spikeinterface/core/node_pipeline.py b/src/spikeinterface/core/node_pipeline.py index 6ecf66918d..13796cb8d9 100644 --- a/src/spikeinterface/core/node_pipeline.py +++ b/src/spikeinterface/core/node_pipeline.py @@ -12,7 +12,7 @@ from spikeinterface.core import BaseRecording, get_chunk_with_margin from spikeinterface.core.job_tools import TimeSeriesChunkExecutor, fix_job_kwargs, _shared_job_kwargs_doc from spikeinterface.core import get_channel_distances -from spikeinterface.core.core_tools import ms_to_samples, samples_to_ms +from spikeinterface.core.core_tools import ms_to_samples, samples_to_ms, _check_zarr_write_is_supported class PipelineNode: @@ -1044,6 +1044,7 @@ def __init__( from spikeinterface.core.zarrextractors import get_default_zarr_compressor + _check_zarr_write_is_supported() if compressor == "default": compressor = get_default_zarr_compressor() self.compressor = compressor diff --git a/src/spikeinterface/core/sortinganalyzer.py b/src/spikeinterface/core/sortinganalyzer.py index cc24463117..37def0c435 100644 --- a/src/spikeinterface/core/sortinganalyzer.py +++ b/src/spikeinterface/core/sortinganalyzer.py @@ -27,6 +27,7 @@ retrieve_importing_provenance, is_path_remote, clean_zarr_folder_name, + _check_zarr_write_is_supported, ) from .sorting_tools import ( generate_unit_ids_for_merge_group, @@ -1013,6 +1014,8 @@ def load_from_binary_folder( def _get_zarr_root(self, mode="r+"): assert mode in ("r+", "a", "r"), "mode must be 'r+', 'a' or 'r'" + if mode != "r": + _check_zarr_write_is_supported() storage_options = self._backend_options.get("storage_options", {}) zarr_root = super_zarr_open(self.folder, mode=mode, storage_options=storage_options) @@ -1034,6 +1037,7 @@ def create_zarr( ) -> "SortingAnalyzer": from .zarrextractors import add_sorting_to_zarr_group, ZarrSortingExtractor + _check_zarr_write_is_supported() is_remote = is_path_remote(folder) if not is_remote: folder = clean_zarr_folder_name(folder) diff --git a/src/spikeinterface/core/template.py b/src/spikeinterface/core/template.py index b661b2d83f..0f0922e10b 100644 --- a/src/spikeinterface/core/template.py +++ b/src/spikeinterface/core/template.py @@ -321,6 +321,10 @@ def add_templates_to_zarr_group(self, zarr_group: "zarr.Group") -> None: to optimize read/write operations for individual units. """ + from .core_tools import _check_zarr_write_is_supported + + _check_zarr_write_is_supported() + # Saves one chunk per unit arrays_chunk = (1, None, None) zarr_group.create_dataset("templates_array", data=self.templates_array, chunks=arrays_chunk) @@ -353,6 +357,9 @@ def to_zarr(self, folder_path: str | Path) -> None: """ import zarr + from .core_tools import _check_zarr_write_is_supported + + _check_zarr_write_is_supported() zarr_group = zarr.open_group(folder_path, mode="w") self.add_templates_to_zarr_group(zarr_group) diff --git a/src/spikeinterface/core/tests/test_analyzer_extension_core.py b/src/spikeinterface/core/tests/test_analyzer_extension_core.py index d96ccf0e37..8436d7c7a3 100644 --- a/src/spikeinterface/core/tests/test_analyzer_extension_core.py +++ b/src/spikeinterface/core/tests/test_analyzer_extension_core.py @@ -11,6 +11,9 @@ from spikeinterface.core.sortinganalyzer import _extension_children, _get_children_dependencies import numpy as np +from spikeinterface.core.core_tools import _is_zarr_write_supported + +analyzer_formats = ("memory", "binary_folder", "zarr") if _is_zarr_write_supported() else ("memory", "binary_folder") def get_sorting_analyzer(cache_folder, format="memory", sparse=True): @@ -51,7 +54,7 @@ def get_sorting_analyzer(cache_folder, format="memory", sparse=True): def _check_result_extension(sorting_analyzer, extension_name, cache_folder): # select unit_ids to several format - for format in ("memory", "binary_folder", "zarr"): + for format in analyzer_formats: # for format in ("memory", ): if format != "memory": if format == "zarr": @@ -72,7 +75,9 @@ def _check_result_extension(sorting_analyzer, extension_name, cache_folder): # print(k, arr.shape) -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) @pytest.mark.parametrize( "sparse", [ @@ -102,7 +107,9 @@ def test_ComputeRandomSpikes(format, sparse, create_cache_folder): _check_result_extension(sorting_analyzer, "random_spikes", cache_folder) -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) @pytest.mark.parametrize("sparse", [True, False]) def test_ComputeWaveforms(format, sparse, create_cache_folder): cache_folder = create_cache_folder @@ -115,6 +122,7 @@ def test_ComputeWaveforms(format, sparse, create_cache_folder): _check_result_extension(sorting_analyzer, "waveforms", cache_folder) +@pytest.mark.requires_zarr_write @pytest.mark.parametrize("sparse", [False, True]) def test_ComputeWaveforms_consistent_across_formats(create_cache_folder, sparse): # Computing waveforms (and templates) on memory / binary_folder / zarr analyzers with the same @@ -170,7 +178,9 @@ def test_ComputeWaveforms_consistent_across_formats(create_cache_folder, sparse) assert np.array_equal(zarr_waveforms, wfs_mem) -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) @pytest.mark.parametrize("sparse", [True, False]) def test_ComputeTemplates(format, sparse, create_cache_folder): cache_folder = create_cache_folder @@ -259,7 +269,9 @@ def test_ComputeTemplates(format, sparse, create_cache_folder): _check_result_extension(sorting_analyzer, "templates", cache_folder) -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) @pytest.mark.parametrize("sparse", [True, False]) def test_ComputeNoiseLevels(format, sparse, create_cache_folder): cache_folder = create_cache_folder diff --git a/src/spikeinterface/core/tests/test_baserecording.py b/src/spikeinterface/core/tests/test_baserecording.py index f2ddc9c5a7..8fd38ce4bf 100644 --- a/src/spikeinterface/core/tests/test_baserecording.py +++ b/src/spikeinterface/core/tests/test_baserecording.py @@ -24,6 +24,7 @@ from spikeinterface.core.testing import check_recordings_equal from spikeinterface.core import generate_recording +from spikeinterface.core.core_tools import _is_zarr_write_supported def test_BaseRecording(create_cache_folder): @@ -364,32 +365,34 @@ def test_BaseRecording(create_cache_folder): rec_2d = rec_3d.planarize(axes="zy") assert np.allclose(rec_2d.get_channel_locations(), locations_3d[:, [2, 1]]) - # test save to zarr - compressor = get_default_zarr_compressor() - rec_zarr = rec2.save(format="zarr", folder=cache_folder / "recording", compressor=compressor) - rec_zarr_loaded = load(cache_folder / "recording.zarr") - # annotations is False because Zarr adds compression ratios - check_recordings_equal(rec2, rec_zarr, return_in_uV=False, check_annotations=False, check_properties=True) - check_recordings_equal( - rec_zarr, rec_zarr_loaded, return_in_uV=False, check_annotations=False, check_properties=True - ) - for annotation_name in rec2.get_annotation_keys(): - assert rec2.get_annotation(annotation_name) == rec_zarr.get_annotation(annotation_name) - assert rec2.get_annotation(annotation_name) == rec_zarr_loaded.get_annotation(annotation_name) - - rec_zarr2 = rec2.save( - format="zarr", folder=cache_folder / "recording_channel_chunk", compressor=compressor, channel_chunk_size=2 - ) - rec_zarr2_loaded = load(cache_folder / "recording_channel_chunk.zarr") - - # annotations is False because Zarr adds compression ratios - check_recordings_equal(rec2, rec_zarr2, return_in_uV=False, check_annotations=False, check_properties=True) - check_recordings_equal( - rec_zarr2, rec_zarr2_loaded, return_in_uV=False, check_annotations=False, check_properties=True - ) - for annotation_name in rec2.get_annotation_keys(): - assert rec2.get_annotation(annotation_name) == rec_zarr2.get_annotation(annotation_name) - assert rec2.get_annotation(annotation_name) == rec_zarr2_loaded.get_annotation(annotation_name) + # TODO: remove once writing to zarr is supported with zarr>=3 + if _is_zarr_write_supported(): + # test save to zarr + compressor = get_default_zarr_compressor() + rec_zarr = rec2.save(format="zarr", folder=cache_folder / "recording", compressor=compressor) + rec_zarr_loaded = load(cache_folder / "recording.zarr") + # annotations is False because Zarr adds compression ratios + check_recordings_equal(rec2, rec_zarr, return_in_uV=False, check_annotations=False, check_properties=True) + check_recordings_equal( + rec_zarr, rec_zarr_loaded, return_in_uV=False, check_annotations=False, check_properties=True + ) + for annotation_name in rec2.get_annotation_keys(): + assert rec2.get_annotation(annotation_name) == rec_zarr.get_annotation(annotation_name) + assert rec2.get_annotation(annotation_name) == rec_zarr_loaded.get_annotation(annotation_name) + + rec_zarr2 = rec2.save( + format="zarr", folder=cache_folder / "recording_channel_chunk", compressor=compressor, channel_chunk_size=2 + ) + rec_zarr2_loaded = load(cache_folder / "recording_channel_chunk.zarr") + + # annotations is False because Zarr adds compression ratios + check_recordings_equal(rec2, rec_zarr2, return_in_uV=False, check_annotations=False, check_properties=True) + check_recordings_equal( + rec_zarr2, rec_zarr2_loaded, return_in_uV=False, check_annotations=False, check_properties=True + ) + for annotation_name in rec2.get_annotation_keys(): + assert rec2.get_annotation(annotation_name) == rec_zarr2.get_annotation(annotation_name) + assert rec2.get_annotation(annotation_name) == rec_zarr2_loaded.get_annotation(annotation_name) def test_json_pickle_equivalence(create_cache_folder): diff --git a/src/spikeinterface/core/tests/test_basesorting.py b/src/spikeinterface/core/tests/test_basesorting.py index 25ba1a3c2d..ec7c2cefb6 100644 --- a/src/spikeinterface/core/tests/test_basesorting.py +++ b/src/spikeinterface/core/tests/test_basesorting.py @@ -25,6 +25,7 @@ from spikeinterface.core.base import BaseExtractor, minimum_spike_dtype, unit_period_dtype from spikeinterface.core.basesorting import LEXSORT_UNIT_COMPACT from spikeinterface.core.testing import check_sorted_arrays_equal, check_sortings_equal +from spikeinterface.core.core_tools import _is_zarr_write_supported def test_BaseSorting(create_cache_folder): @@ -143,16 +144,18 @@ def test_BaseSorting(create_cache_folder): del sorting6 del sorting5 - # test save to zarr - # compressor = get_default_zarr_compressor() - sorting_zarr = sorting.save(format="zarr", folder=cache_folder / "sorting.zarr") - sorting_zarr_loaded = load(cache_folder / "sorting.zarr") - # annotations is False because Zarr adds compression ratios - check_sortings_equal(sorting, sorting_zarr, check_annotations=False, check_properties=True) - check_sortings_equal(sorting_zarr, sorting_zarr_loaded, check_annotations=False, check_properties=True) - for annotation_name in sorting.get_annotation_keys(): - assert sorting.get_annotation(annotation_name) == sorting_zarr.get_annotation(annotation_name) - assert sorting.get_annotation(annotation_name) == sorting_zarr_loaded.get_annotation(annotation_name) + # TODO: remove once writing to zarr is supported with zarr>=3 + if _is_zarr_write_supported(): + # test save to zarr + # compressor = get_default_zarr_compressor() + sorting_zarr = sorting.save(format="zarr", folder=cache_folder / "sorting.zarr") + sorting_zarr_loaded = load(cache_folder / "sorting.zarr") + # annotations is False because Zarr adds compression ratios + check_sortings_equal(sorting, sorting_zarr, check_annotations=False, check_properties=True) + check_sortings_equal(sorting_zarr, sorting_zarr_loaded, check_annotations=False, check_properties=True) + for annotation_name in sorting.get_annotation_keys(): + assert sorting.get_annotation(annotation_name) == sorting_zarr.get_annotation(annotation_name) + assert sorting.get_annotation(annotation_name) == sorting_zarr_loaded.get_annotation(annotation_name) def _make_sorting_with_shuffled_ties(num_units, num_segments, seed=42): diff --git a/src/spikeinterface/core/tests/test_loading.py b/src/spikeinterface/core/tests/test_loading.py index 257895fcc2..364c5bd910 100644 --- a/src/spikeinterface/core/tests/test_loading.py +++ b/src/spikeinterface/core/tests/test_loading.py @@ -1,4 +1,5 @@ import pytest +import zarr import numpy as np from spikeinterface import ( @@ -89,7 +90,7 @@ def generate_motion_object(): return motion -@pytest.mark.parametrize("output_format", ["binary", "zarr"]) +@pytest.mark.parametrize("output_format", ["binary", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)]) def test_load_binary_recording(generate_recording_sorting, tmp_path, output_format): rec, _ = generate_recording_sorting _ = rec.save(folder=tmp_path / "test_recording", format=output_format, overwrite=True) @@ -102,7 +103,7 @@ def test_load_binary_recording(generate_recording_sorting, tmp_path, output_form check_recordings_equal(rec, rec_loaded) -@pytest.mark.parametrize("output_format", ["numpy_folder", "zarr"]) +@pytest.mark.parametrize("output_format", ["numpy_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)]) def test_load_binary_sorting(generate_recording_sorting, tmp_path, output_format): _, sort = generate_recording_sorting _ = sort.save(folder=tmp_path / "test_sorting", format=output_format, overwrite=True) @@ -140,7 +141,9 @@ def test_load_ext_extractors(generate_recording_sorting, tmp_path, extension): check_sortings_equal(sort, sort_loaded, check_properties=False) -@pytest.mark.parametrize("output_format", ["binary_folder", "zarr"]) +@pytest.mark.parametrize( + "output_format", ["binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) def test_load_sorting_analyzer(generate_sorting_analyzer, tmp_path, output_format): analyzer = generate_sorting_analyzer _ = analyzer.save_as(folder=tmp_path / "analyzer", format=output_format) @@ -157,6 +160,7 @@ def test_load_sorting_analyzer(generate_sorting_analyzer, tmp_path, output_forma assert ext in analyzer_loaded.extensions +@pytest.mark.requires_zarr_write def test_load_templates(tmp_path, generate_templates_object): templates = generate_templates_object templates_dict = templates.to_dict() @@ -212,6 +216,8 @@ def test_remote_recording(): @pytest.mark.streaming_extractors @pytest.mark.skipif(not HAVE_S3, reason="s3fs not installed") +# TODO: remove once streaming zarr is supported with zarr>=3 +@pytest.mark.skipif(int(zarr.__version__.split(".")[0]) >= 3, reason="Streaming zarr is not supported yet with zarr>=3") def test_remote_analyzer(): s3_path = "s3://spikeinterface-sorting-analyzer-test/analyzer_remote_test.zarr/" analyzer = load(s3_path) diff --git a/src/spikeinterface/core/tests/test_node_pipeline.py b/src/spikeinterface/core/tests/test_node_pipeline.py index a721fc3367..533966d703 100644 --- a/src/spikeinterface/core/tests/test_node_pipeline.py +++ b/src/spikeinterface/core/tests/test_node_pipeline.py @@ -16,6 +16,7 @@ ExtractDenseWaveforms, sorting_to_peaks, ) +from spikeinterface.core.core_tools import _is_zarr_write_supported class AmplitudeExtractionNode(PipelineNode): @@ -180,34 +181,36 @@ def test_run_node_pipeline(cache_folder_creation): assert np.array_equal(denoised_waveforms_rms, denoised_waveforms_rms2) assert np.array_equal(denoised_waveforms_rms2, denoised_waveforms_rms3) - # gather zarr mode - import zarr - - zarr_folder = cache_folder / f"pipeline_folder_{loop}.zarr" - if zarr_folder.is_dir(): - shutil.rmtree(zarr_folder) - output = run_node_pipeline( - recording, - nodes, - job_kwargs, - gather_mode="zarr", - dest=zarr_folder, - names=["amplitudes", "waveforms_rms", "denoised_waveforms_rms"], - ) - amplitudes_z, waveforms_rms_z, denoised_waveforms_rms_z = output - - # values must match the memory gather - assert np.array_equal(amplitudes, amplitudes_z[:]) - assert np.array_equal(waveforms_rms, waveforms_rms_z[:]) - assert np.array_equal(denoised_waveforms_rms, denoised_waveforms_rms_z[:]) - - # arrays must be persisted on disk and re-openable - zarr_root = zarr.open(str(zarr_folder), mode="r") - for name in ("amplitudes", "waveforms_rms", "denoised_waveforms_rms"): - assert name in zarr_root - assert np.array_equal(amplitudes, zarr_root["amplitudes"][:]) - assert np.array_equal(waveforms_rms, zarr_root["waveforms_rms"][:]) - assert np.array_equal(denoised_waveforms_rms, zarr_root["denoised_waveforms_rms"][:]) + # TODO: remove once writing to zarr is supported with zarr>=3 + if _is_zarr_write_supported(): + # gather zarr mode + import zarr + + zarr_folder = cache_folder / f"pipeline_folder_{loop}.zarr" + if zarr_folder.is_dir(): + shutil.rmtree(zarr_folder) + output = run_node_pipeline( + recording, + nodes, + job_kwargs, + gather_mode="zarr", + dest=zarr_folder, + names=["amplitudes", "waveforms_rms", "denoised_waveforms_rms"], + ) + amplitudes_z, waveforms_rms_z, denoised_waveforms_rms_z = output + + # values must match the memory gather + assert np.array_equal(amplitudes, amplitudes_z[:]) + assert np.array_equal(waveforms_rms, waveforms_rms_z[:]) + assert np.array_equal(denoised_waveforms_rms, denoised_waveforms_rms_z[:]) + + # arrays must be persisted on disk and re-openable + zarr_root = zarr.open(str(zarr_folder), mode="r") + for name in ("amplitudes", "waveforms_rms", "denoised_waveforms_rms"): + assert name in zarr_root + assert np.array_equal(amplitudes, zarr_root["amplitudes"][:]) + assert np.array_equal(waveforms_rms, zarr_root["waveforms_rms"][:]) + assert np.array_equal(denoised_waveforms_rms, zarr_root["denoised_waveforms_rms"][:]) # gather npy mode with an explicit list of file paths (final location) npy_files_folder = cache_folder / f"pipeline_npy_files_{loop}" @@ -232,36 +235,38 @@ def test_run_node_pipeline(cache_folder_creation): assert np.array_equal(waveforms_rms, waveforms_rms_f) assert np.array_equal(denoised_waveforms_rms, denoised_waveforms_rms_f) - # gather zarr mode with an explicit list of dataset paths, created on the fly - # inside an existing store (final location, e.g. an analyzer extension group) - datasets_store = cache_folder / f"pipeline_zarr_datasets_{loop}.zarr" - if datasets_store.is_dir(): - shutil.rmtree(datasets_store) - # pre-existing store that must not be wiped - root = zarr.open(str(datasets_store), mode="w") - root.attrs["preexisting"] = True - dataset_paths = [ - datasets_store / "extensions" / "amplitudes", - datasets_store / "extensions" / "waveforms_rms", - datasets_store / "extensions" / "denoised_waveforms_rms", - ] - output = run_node_pipeline( - recording, - nodes, - job_kwargs, - gather_mode="zarr", - dest=dataset_paths, - ) - amplitudes_d, waveforms_rms_d, denoised_waveforms_rms_d = output - assert np.array_equal(amplitudes, amplitudes_d[:]) - assert np.array_equal(waveforms_rms, waveforms_rms_d[:]) - assert np.array_equal(denoised_waveforms_rms, denoised_waveforms_rms_d[:]) - # data must be persisted at the passed final location and the store not wiped - root_reopen = zarr.open(str(datasets_store), mode="r") - assert root_reopen.attrs.get("preexisting", False) - assert np.array_equal(amplitudes, root_reopen["extensions"]["amplitudes"][:]) - assert np.array_equal(waveforms_rms, root_reopen["extensions"]["waveforms_rms"][:]) - assert np.array_equal(denoised_waveforms_rms, root_reopen["extensions"]["denoised_waveforms_rms"][:]) + # TODO: remove once writing to zarr is supported with zarr>=3 + if _is_zarr_write_supported(): + # gather zarr mode with an explicit list of dataset paths, created on the fly + # inside an existing store (final location, e.g. an analyzer extension group) + datasets_store = cache_folder / f"pipeline_zarr_datasets_{loop}.zarr" + if datasets_store.is_dir(): + shutil.rmtree(datasets_store) + # pre-existing store that must not be wiped + root = zarr.open(str(datasets_store), mode="w") + root.attrs["preexisting"] = True + dataset_paths = [ + datasets_store / "extensions" / "amplitudes", + datasets_store / "extensions" / "waveforms_rms", + datasets_store / "extensions" / "denoised_waveforms_rms", + ] + output = run_node_pipeline( + recording, + nodes, + job_kwargs, + gather_mode="zarr", + dest=dataset_paths, + ) + amplitudes_d, waveforms_rms_d, denoised_waveforms_rms_d = output + assert np.array_equal(amplitudes, amplitudes_d[:]) + assert np.array_equal(waveforms_rms, waveforms_rms_d[:]) + assert np.array_equal(denoised_waveforms_rms, denoised_waveforms_rms_d[:]) + # data must be persisted at the passed final location and the store not wiped + root_reopen = zarr.open(str(datasets_store), mode="r") + assert root_reopen.attrs.get("preexisting", False) + assert np.array_equal(amplitudes, root_reopen["extensions"]["amplitudes"][:]) + assert np.array_equal(waveforms_rms, root_reopen["extensions"]["waveforms_rms"][:]) + assert np.array_equal(denoised_waveforms_rms, root_reopen["extensions"]["denoised_waveforms_rms"][:]) # Test pickle mechanism for node in nodes: @@ -271,6 +276,7 @@ def test_run_node_pipeline(cache_folder_creation): unpickled_node = pickle.loads(pickled_node) +@pytest.mark.requires_zarr_write def test_gather_to_zarr_chunking(tmp_path): # the zarr chunk size along the first axis must be picked from a byte target (not from the # size of the first gathered buffer), so it stays sensible for billions of spikes and never @@ -319,6 +325,7 @@ def test_gather_to_zarr_chunking(tmp_path): assert np.array_equal(waveforms[:], waveforms2[:]) +@pytest.mark.requires_zarr_write def test_gather_to_zarr_chunk_bytes_per_name(tmp_path): # `zarr_target_chunk_bytes` can also be a dict to use a different byte target per array recording, sorting = generate_ground_truth_recording(num_channels=8, num_units=5, durations=[20.0], seed=7) diff --git a/src/spikeinterface/core/tests/test_sortinganalyzer.py b/src/spikeinterface/core/tests/test_sortinganalyzer.py index 856612a6cf..84440c42a1 100644 --- a/src/spikeinterface/core/tests/test_sortinganalyzer.py +++ b/src/spikeinterface/core/tests/test_sortinganalyzer.py @@ -25,6 +25,9 @@ # to test basespikevectorextension with node pipeline from spikeinterface.core.node_pipeline import SpikeRetriever from spikeinterface.core.tests.test_node_pipeline import AmplitudeExtractionNode +from spikeinterface.core.core_tools import _is_zarr_write_supported + +analyzer_formats = ("memory", "binary_folder", "zarr") if _is_zarr_write_supported() else ("memory", "binary_folder") def get_dataset(): @@ -138,6 +141,7 @@ def test_SortingAnalyzer_binary_folder(tmp_path, dataset): assert "number" in sorting_analyzer_reloded.sorting.get_property_keys() +@pytest.mark.requires_zarr_write def test_SortingAnalyzer_zarr(tmp_path, dataset): recording, sorting = dataset recording = recording.save(folder=tmp_path / "recording_zarr") @@ -309,20 +313,22 @@ def test_load_without_runtime_info(tmp_path, dataset): with pytest.warns(UserWarning): sorting_analyzer = load_sorting_analyzer(folder, format="auto") - # zarr - folder = tmp_path / "test_SortingAnalyzer_run_info.zarr" - sorting_analyzer = create_sorting_analyzer( - sorting, recording, format="zarr", folder=folder, sparse=False, sparsity=None - ) - sorting_analyzer.compute(extensions) - # remove run_info from attrs to mimic a previous version of spikeinterface - root = sorting_analyzer._get_zarr_root(mode="r+") - for ext in extensions: - del root["extensions"][ext].attrs["run_info"] - zarr.consolidate_metadata(root.store) - # should raise a warning for missing run_info - with pytest.warns(UserWarning): - sorting_analyzer = load_sorting_analyzer(folder, format="auto") + # TODO: remove once writing to zarr is supported with zarr>=3 + if _is_zarr_write_supported(): + # zarr + folder = tmp_path / "test_SortingAnalyzer_run_info.zarr" + sorting_analyzer = create_sorting_analyzer( + sorting, recording, format="zarr", folder=folder, sparse=False, sparsity=None + ) + sorting_analyzer.compute(extensions) + # remove run_info from attrs to mimic a previous version of spikeinterface + root = sorting_analyzer._get_zarr_root(mode="r+") + for ext in extensions: + del root["extensions"][ext].attrs["run_info"] + zarr.consolidate_metadata(root.store) + # should raise a warning for missing run_info + with pytest.warns(UserWarning): + sorting_analyzer = load_sorting_analyzer(folder, format="auto") def test_SortingAnalyzer_tmp_recording(dataset): @@ -366,7 +372,7 @@ def test_SortingAnalyzer_interleaved_probegroup(dataset): assert np.array_equal(recording.get_channel_locations(), sorting_analyzer.get_channel_locations()) -@pytest.mark.parametrize("format", ["binary_folder", "zarr"]) +@pytest.mark.parametrize("format", ["binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)]) def test_load_in_lazy_mode(tmp_path, dataset, format): recording, sorting = dataset @@ -447,7 +453,7 @@ def _check_sorting_analyzers(sorting_analyzer, original_sorting, cache_folder): assert sorting_analyzer.has_recording() # save to several format - for format in ("memory", "binary_folder", "zarr"): + for format in analyzer_formats: if format != "memory": if format == "zarr": folder = cache_folder / f"test_SortingAnalyzer_save_as_{format}.zarr" @@ -477,7 +483,7 @@ def _check_sorting_analyzers(sorting_analyzer, original_sorting, cache_folder): assert sorting_analyzer2.sparsity == sorting_analyzer.sparsity # select unit_ids to several format - for format in ("memory", "binary_folder", "zarr"): + for format in analyzer_formats: if format != "memory": if format == "zarr": folder = cache_folder / f"test_SortingAnalyzer_select_units_with_{format}.zarr" @@ -858,7 +864,9 @@ def _compute_reference_pipeline_data(dataset): return analyzer.get_extension("dummy_pipeline").get_data() -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) @pytest.mark.parametrize("lazy", [True, False]) def test_compute_pipeline_extension_gather_to_disk_lazy(tmp_path, dataset, format, lazy): """ @@ -923,7 +931,7 @@ def test_compute_pipeline_extension_gather_to_disk_lazy(tmp_path, dataset, forma assert np.array_equal(load_sorting_analyzer(folder).get_extension("dummy_pipeline").get_data(), amp_ref) -@pytest.mark.parametrize("format", ["binary_folder", "zarr"]) +@pytest.mark.parametrize("format", ["binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)]) def test_compute_pipeline_extension_save_false(tmp_path, dataset, format): """ With save=False on a disk-backed analyzer, node-pipeline extensions are computed in memory @@ -945,7 +953,9 @@ def test_compute_pipeline_extension_save_false(tmp_path, dataset, format): assert not analyzer_reloaded.has_extension("dummy_pipeline") -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) @pytest.mark.parametrize("lazy", [True, False]) def test_compute_one_pipeline_extension_gather_to_disk(tmp_path, dataset, format, lazy): """ diff --git a/src/spikeinterface/core/tests/test_template_class.py b/src/spikeinterface/core/tests/test_template_class.py index 953527c8d5..20f88817cb 100644 --- a/src/spikeinterface/core/tests/test_template_class.py +++ b/src/spikeinterface/core/tests/test_template_class.py @@ -115,6 +115,7 @@ def test_initialization_fail_with_dense_templates(): template = generate_test_template(template_type="sparse_with_dense_templates") +@pytest.mark.requires_zarr_write @pytest.mark.parametrize("is_in_uV", [True, False]) @pytest.mark.parametrize("template_type", ["dense", "sparse"]) def test_save_and_load_zarr(template_type, is_in_uV, tmp_path): diff --git a/src/spikeinterface/core/tests/test_time_handling.py b/src/spikeinterface/core/tests/test_time_handling.py index 2f3cc1a6bf..4a08a1c7c8 100644 --- a/src/spikeinterface/core/tests/test_time_handling.py +++ b/src/spikeinterface/core/tests/test_time_handling.py @@ -114,7 +114,7 @@ def test_has_time_vector(self, time_vector_recording): assert raw_recording.has_time_vector(segment_idx) is False assert times_recording.has_time_vector(segment_idx) is True - @pytest.mark.parametrize("mode", ["binary", "zarr"]) + @pytest.mark.parametrize("mode", ["binary", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)]) @pytest.mark.parametrize("fixture_name", ["time_vector_recording", "t_start_recording"]) def test_times_propagated_to_save_folder(self, request, fixture_name, mode, tmp_path): """ @@ -375,7 +375,7 @@ def test_save_and_load_time_shift(self, request, fixture_name, tmp_path): times_recording.get_times(segment_index=idx), loaded_recording.get_times(segment_index=idx) ) - @pytest.mark.parametrize("save_format", ["binary", "zarr"]) + @pytest.mark.parametrize("save_format", ["binary", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)]) def test_shift_times_after_load(self, request, save_format, tmp_path): """ Shift times on a recording loaded from disk as a read-only np.memmap diff --git a/src/spikeinterface/core/tests/test_waveform_tools.py b/src/spikeinterface/core/tests/test_waveform_tools.py index 6623c92046..c2750774b6 100644 --- a/src/spikeinterface/core/tests/test_waveform_tools.py +++ b/src/spikeinterface/core/tests/test_waveform_tools.py @@ -157,6 +157,7 @@ def test_waveform_tools(create_cache_folder): _check_all_wf_equal(list_wfs_sparse) +@pytest.mark.requires_zarr_write @pytest.mark.parametrize("sparse", [False, True]) def test_extract_waveforms_to_single_buffer_zarr(tmp_path, sparse): # the "zarr" mode writes waveforms directly to a zarr dataset. Workers return their block and @@ -210,6 +211,7 @@ def test_extract_waveforms_to_single_buffer_zarr(tmp_path, sparse): assert np.array_equal(reference, reloaded[:]) +@pytest.mark.requires_zarr_write def test_waveforms_at_segment_borders(tmp_path): # spikes near the segment borders are partially filled: samples outside of the segment are 0 from spikeinterface.core import NumpySorting diff --git a/src/spikeinterface/core/tests/test_zarrextractors.py b/src/spikeinterface/core/tests/test_zarrextractors.py index 6278541698..4d546b4c4a 100644 --- a/src/spikeinterface/core/tests/test_zarrextractors.py +++ b/src/spikeinterface/core/tests/test_zarrextractors.py @@ -16,6 +16,7 @@ ) +@pytest.mark.requires_zarr_write def test_zarr_compression_options(tmp_path): from numcodecs import Blosc, Delta, FixedScaleOffset @@ -59,6 +60,7 @@ def test_zarr_compression_options(tmp_path): assert rec_other._root["times_seg0"].filters == other_filters2 +@pytest.mark.requires_zarr_write def test_ZarrSortingExtractor(tmp_path): np_sorting = generate_sorting() diff --git a/src/spikeinterface/core/zarrextractors.py b/src/spikeinterface/core/zarrextractors.py index 06f887871b..7a90089edd 100644 --- a/src/spikeinterface/core/zarrextractors.py +++ b/src/spikeinterface/core/zarrextractors.py @@ -11,7 +11,13 @@ from .baserecording import BaseRecording, BaseRecordingSegment from .basesorting import BaseSorting, SpikeVectorSortingSegment from .job_tools import split_job_kwargs -from .core_tools import define_function_from_class, check_json, retrieve_importing_provenance, is_path_remote +from .core_tools import ( + define_function_from_class, + check_json, + retrieve_importing_provenance, + is_path_remote, + _check_zarr_write_is_supported, +) from .time_series_tools import _write_time_series_to_zarr @@ -78,9 +84,11 @@ def super_zarr_open(folder_path: str | Path, mode: str = "r", storage_options: d else: if not Path(folder_path).is_dir(): raise ValueError(f"Folder {folder_path} does not exist") + # zarr>=3 refuses storage_options for a local path, even an empty dict + local_kwargs = dict(storage_options=storage_options) if storage_options else dict() for open_func in open_funcs: try: - root = open_func(str(folder_path), mode=mode, storage_options=storage_options) + root = open_func(str(folder_path), mode=mode, **local_kwargs) break except Exception as e: exception = e @@ -642,6 +650,7 @@ def create_zarr_path_for_write(folder_path: str | Path, overwrite: bool = False) folder_path : str or Path Path to the zarr root file """ + _check_zarr_write_is_supported() if not is_path_remote(folder_path): folder_path = Path(folder_path) folder_path = folder_path.with_suffix(".zarr") diff --git a/src/spikeinterface/extractors/nwbextractors.py b/src/spikeinterface/extractors/nwbextractors.py index 13f1d2d605..0d1946d39a 100644 --- a/src/spikeinterface/extractors/nwbextractors.py +++ b/src/spikeinterface/extractors/nwbextractors.py @@ -333,7 +333,9 @@ def _find_neurodata_type_from_backend(group, path="", result=None, neurodata_typ if result is None: result = [] - for neurodata_name, value in group.items(): + # zarr>=3 groups have no `items()`, `keys()` works for h5py and both zarr versions + for neurodata_name in group.keys(): + value = group[neurodata_name] # Check if it's a group and if it has the neurodata_type if isinstance(value, group_class): current_path = f"{path}/{neurodata_name}" if path else neurodata_name @@ -354,8 +356,8 @@ def _retrieve_electrodes_indices_from_electrical_series_backend(open_file, elect if backend == "zarr": import zarr - # links must be resolved - zarr_links = electrical_series.attrs["zarr_link"] + # links must be resolved, hdmf-zarr>=0.14 stores them under "_LINKS" instead of "zarr_link" + zarr_links = electrical_series.attrs.get("zarr_link", electrical_series.attrs.get("_LINKS")) electrodes_path = None for zarr_link in zarr_links: if zarr_link["name"] == "electrodes": @@ -1642,7 +1644,9 @@ def _find_timeseries_from_backend(group, path="", result=None, backend="hdf5"): if result is None: result = [] - for name, value in group.items(): + # zarr>=3 groups have no `items()`, `keys()` works for h5py and both zarr versions + for name in group.keys(): + value = group[name] if isinstance(value, group_class): current_path = f"{path}/{name}" if path else name if value.attrs.get("neurodata_type") == "TimeSeries": diff --git a/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py b/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py index 161040cbaf..9a7b2c0c67 100644 --- a/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py +++ b/src/spikeinterface/extractors/tests/test_nwbextractors_streaming.py @@ -3,12 +3,18 @@ import pytest import numpy as np +import zarr from spikeinterface import load from spikeinterface.core.testing import check_recordings_equal from spikeinterface.core.testing import check_recordings_equal, check_sortings_equal from spikeinterface.extractors.extractor_classes import NwbRecordingExtractor, NwbSortingExtractor +# TODO: remove once streaming zarr is supported with zarr>=3 +skip_zarr_3 = pytest.mark.skipif( + int(zarr.__version__.split(".")[0]) >= 3, reason="Streaming zarr is not supported yet with zarr>=3" +) + @pytest.mark.streaming_extractors @pytest.mark.parametrize("cache", [True, False]) # Test with and without cache @@ -185,6 +191,7 @@ def test_sorting_s3_nwb_remfile(tmp_path): @pytest.mark.streaming_extractors +@skip_zarr_3 def test_sorting_s3_nwb_zarr(tmp_path): file_path = ( "s3://aind-open-data/ecephys_625749_2022-08-03_15-15-06_nwb_2023-05-16_16-34-55/" diff --git a/src/spikeinterface/generation/tests/test_template_database.py b/src/spikeinterface/generation/tests/test_template_database.py index 78dfb7a6ec..3a3034acf3 100644 --- a/src/spikeinterface/generation/tests/test_template_database.py +++ b/src/spikeinterface/generation/tests/test_template_database.py @@ -1,3 +1,6 @@ +import pytest +import zarr + from spikeinterface.core.template import Templates from spikeinterface.generation import ( @@ -7,7 +10,13 @@ query_templates_from_database, ) +# TODO: remove once probeinterface can read zarr>=3 groups (Probe.from_zarr_group calls len() on a zarr array) +skip_zarr_3 = pytest.mark.skipif( + int(zarr.__version__.split(".")[0]) >= 3, reason="Reading templates from zarr is not supported yet with zarr>=3" +) + +@skip_zarr_3 def test_fetch_template_object_from_database(): available_datasets = list_available_datasets_in_template_database() @@ -31,6 +40,7 @@ def test_fetch_templates_database_info(): assert "dataset" in templates_info.columns +@skip_zarr_3 def test_query_templates_from_database(): templates_info = fetch_templates_database_info() diff --git a/src/spikeinterface/postprocessing/tests/common_extension_tests.py b/src/spikeinterface/postprocessing/tests/common_extension_tests.py index 58e2092149..fb7770befe 100644 --- a/src/spikeinterface/postprocessing/tests/common_extension_tests.py +++ b/src/spikeinterface/postprocessing/tests/common_extension_tests.py @@ -9,6 +9,7 @@ estimate_sparsity, ) from spikeinterface.core.sortinganalyzer import get_extension_class +from spikeinterface.core.core_tools import _is_zarr_write_supported extensions_which_allow_unit_ids = ["unit_locations"] extensions_with_unit_by_unit_data = ["correlograms", "template_similarity"] @@ -241,8 +242,10 @@ def run_extension_tests(self, extension_class, params): of interest with the passed parameters. Will perform tests for sparsity and format. """ + # TODO: remove once writing to zarr is supported with zarr>=3 + formats = ("memory", "binary_folder", "zarr") if _is_zarr_write_supported() else ("memory", "binary_folder") for sparse in (True, False): - for format in ("memory", "binary_folder", "zarr"): + for format in formats: print("sparse", sparse, format) sorting_analyzer = self._prepare_sorting_analyzer( format, sparse, extension_class, extension_params=params diff --git a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py index 7b53f24f29..6ae72e374d 100644 --- a/src/spikeinterface/postprocessing/tests/test_multi_extensions.py +++ b/src/spikeinterface/postprocessing/tests/test_multi_extensions.py @@ -133,7 +133,9 @@ def dataset_to_split(create_cache_folder): @pytest.mark.parametrize("lazy", [False, True]) @pytest.mark.parametrize("sparse", [False, True]) -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) def test_SortingAnalyzer_merge_all_extensions(dataset_to_merge, lazy, sparse, format, tmp_path): if format == "memory" and lazy: pytest.skip("lazy has no effect for format='memory' (nothing on disk to load lazily)") @@ -265,7 +267,9 @@ def test_SortingAnalyzer_merge_all_extensions(dataset_to_merge, lazy, sparse, fo @pytest.mark.parametrize("lazy", [False, True]) @pytest.mark.parametrize("sparse", [False, True]) -@pytest.mark.parametrize("format", ["memory", "binary_folder", "zarr"]) +@pytest.mark.parametrize( + "format", ["memory", "binary_folder", pytest.param("zarr", marks=pytest.mark.requires_zarr_write)] +) def test_SortingAnalyzer_split_all_extensions(dataset_to_split, lazy, sparse, format, tmp_path): if format == "memory" and lazy: pytest.skip("lazy has no effect for format='memory' (nothing on disk to load lazily)") diff --git a/src/spikeinterface/preprocessing/tests/test_pipeline.py b/src/spikeinterface/preprocessing/tests/test_pipeline.py index 4a62bc0088..7b53b0ab92 100644 --- a/src/spikeinterface/preprocessing/tests/test_pipeline.py +++ b/src/spikeinterface/preprocessing/tests/test_pipeline.py @@ -2,6 +2,7 @@ from spikeinterface.core.testing import check_recordings_equal from spikeinterface.core import create_sorting_analyzer +from spikeinterface.core.core_tools import _is_zarr_write_supported from spikeinterface.generation import generate_recording, generate_ground_truth_recording from spikeinterface.preprocessing import ( apply_preprocessing_pipeline, @@ -218,11 +219,13 @@ def test_loading_from_analyzer(create_cache_folder): pp_recording_from_binary = apply_preprocessing_pipeline(recording, pp_list_from_binary) check_recordings_equal(pp_recording, pp_recording_from_binary) - analyzer_zarr_folder = cache_folder / "zarr_format.zarr" - _ = create_sorting_analyzer(sorting=sorting, recording=pp_recording, format="zarr", folder=analyzer_zarr_folder) - pp_list_from_zarr = get_preprocessing_list_from_analyzer(analyzer_zarr_folder) - pp_recording_from_zarr = apply_preprocessing_pipeline(recording, pp_list_from_zarr) - check_recordings_equal(pp_recording, pp_recording_from_zarr) + # TODO: remove once writing to zarr is supported with zarr>=3 + if _is_zarr_write_supported(): + analyzer_zarr_folder = cache_folder / "zarr_format.zarr" + _ = create_sorting_analyzer(sorting=sorting, recording=pp_recording, format="zarr", folder=analyzer_zarr_folder) + pp_list_from_zarr = get_preprocessing_list_from_analyzer(analyzer_zarr_folder) + pp_recording_from_zarr = apply_preprocessing_pipeline(recording, pp_list_from_zarr) + check_recordings_equal(pp_recording, pp_recording_from_zarr) def test_pipeline_recording_arg_substitution(create_cache_folder): diff --git a/src/spikeinterface/sorters/internal/lupin.py b/src/spikeinterface/sorters/internal/lupin.py index f6a055c5c8..6d79bb036d 100644 --- a/src/spikeinterface/sorters/internal/lupin.py +++ b/src/spikeinterface/sorters/internal/lupin.py @@ -11,6 +11,7 @@ ) from spikeinterface.core.job_tools import fix_job_kwargs +from spikeinterface.core.core_tools import _check_zarr_write_is_supported from spikeinterface.preprocessing import bandpass_filter, common_reference, zscore, whiten from spikeinterface.core.base import minimum_spike_dtype @@ -124,6 +125,10 @@ def get_sorter_version(cls): @classmethod def _run_from_folder(cls, sorter_output_folder, params, verbose): + # TODO: remove once writing to zarr is supported with zarr>=3 + if params["save_array"]: + _check_zarr_write_is_supported() + from spikeinterface.sortingcomponents.tools import get_prototype_and_waveforms_from_recording from spikeinterface.sortingcomponents.matching import find_spikes_from_templates from spikeinterface.sortingcomponents.peak_detection import detect_peaks diff --git a/src/spikeinterface/sorters/internal/tests/test_lupin.py b/src/spikeinterface/sorters/internal/tests/test_lupin.py index df2666be1d..c4eb788a6d 100644 --- a/src/spikeinterface/sorters/internal/tests/test_lupin.py +++ b/src/spikeinterface/sorters/internal/tests/test_lupin.py @@ -1,5 +1,7 @@ import unittest +import pytest + from spikeinterface.sorters.tests.common_tests import SorterCommonTestSuite from spikeinterface.sorters import LupinSorter, run_sorter @@ -10,6 +12,11 @@ class LupinSorterCommonTestSuite(SorterCommonTestSuite, unittest.TestCase): SorterClass = LupinSorter + # TODO: remove once writing to zarr is supported with zarr>=3 (save_array writes the templates to zarr) + @pytest.mark.requires_zarr_write + def test_with_run(self): + super().test_with_run() + if __name__ == "__main__": test = LupinSorterCommonTestSuite() diff --git a/src/spikeinterface/sorters/internal/tests/test_tridesclous2.py b/src/spikeinterface/sorters/internal/tests/test_tridesclous2.py index 01284d7ee6..cb6ed09ce4 100644 --- a/src/spikeinterface/sorters/internal/tests/test_tridesclous2.py +++ b/src/spikeinterface/sorters/internal/tests/test_tridesclous2.py @@ -1,5 +1,7 @@ import unittest +import pytest + from spikeinterface.sorters.tests.common_tests import SorterCommonTestSuite from spikeinterface.sorters import Tridesclous2Sorter, run_sorter @@ -10,6 +12,11 @@ class Tridesclous2SorterCommonTestSuite(SorterCommonTestSuite, unittest.TestCase): SorterClass = Tridesclous2Sorter + # TODO: remove once writing to zarr is supported with zarr>=3 (save_array writes the templates to zarr) + @pytest.mark.requires_zarr_write + def test_with_run(self): + super().test_with_run() + @unittest.skip("performance reason") def test_with_numpy_gather(self): recording = self.recording diff --git a/src/spikeinterface/sorters/internal/tridesclous2.py b/src/spikeinterface/sorters/internal/tridesclous2.py index d5c66f35be..55d377d4db 100644 --- a/src/spikeinterface/sorters/internal/tridesclous2.py +++ b/src/spikeinterface/sorters/internal/tridesclous2.py @@ -15,6 +15,7 @@ ) from spikeinterface.core.job_tools import fix_job_kwargs +from spikeinterface.core.core_tools import _check_zarr_write_is_supported from spikeinterface.preprocessing import bandpass_filter, common_reference, zscore, whiten from spikeinterface.core.base import minimum_spike_dtype @@ -118,6 +119,10 @@ def get_sorter_version(cls): @classmethod def _run_from_folder(cls, sorter_output_folder, params, verbose): + # TODO: remove once writing to zarr is supported with zarr>=3 + if params["save_array"]: + _check_zarr_write_is_supported() + from spikeinterface.sortingcomponents.matching import find_spikes_from_templates from spikeinterface.sortingcomponents.peak_detection import detect_peaks from spikeinterface.sortingcomponents.peak_selection import select_peaks diff --git a/src/spikeinterface/sorters/tests/test_launcher.py b/src/spikeinterface/sorters/tests/test_launcher.py index 6619431866..35db70e677 100644 --- a/src/spikeinterface/sorters/tests/test_launcher.py +++ b/src/spikeinterface/sorters/tests/test_launcher.py @@ -52,6 +52,7 @@ def job_list(create_cache_folder): return get_job_list(folder) +@pytest.mark.requires_zarr_write def test_run_sorter_jobs_loop(job_list): sortings = run_sorter_jobs(job_list, engine="loop", return_output=True) print(sortings) @@ -201,6 +202,7 @@ def test_run_sorter_jobs_slurm_kwargs(mocker, tmp_path, job_list): assert str(tmp_script_folder) in mock_subprocess_run.call_args_list[-1].args[0][5] +@pytest.mark.requires_zarr_write def test_run_sorter_by_property(create_cache_folder): cache_folder = create_cache_folder working_folder1 = cache_folder / "test_run_sorter_by_property_1" diff --git a/src/spikeinterface/sorters/tests/test_runsorter.py b/src/spikeinterface/sorters/tests/test_runsorter.py index c07cd94c9b..1316430e55 100644 --- a/src/spikeinterface/sorters/tests/test_runsorter.py +++ b/src/spikeinterface/sorters/tests/test_runsorter.py @@ -24,6 +24,7 @@ def generate_recording(create_cache_folder): return _generate_recording(create_cache_folder) +@pytest.mark.requires_zarr_write @pytest.mark.xfail( platform.system() == "Windows" and parse(platform.python_version()) > parse("3.12"), reason="3rd parth threadpoolctl issue: OSError('GetModuleFileNameEx failed')",