From da68ffb75c628c127ec02e5a3914a0fdbe4dcbdf Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 7 Oct 2026 16:58:59 -0600 Subject: [PATCH 1/8] Filter one channel at a time to reduce peak memory --- src/spikeinterface/preprocessing/filter.py | 54 +++++++++++++--------- 1 file changed, 31 insertions(+), 23 deletions(-) diff --git a/src/spikeinterface/preprocessing/filter.py b/src/spikeinterface/preprocessing/filter.py index 2cf3e47c93..e81a158fb6 100644 --- a/src/spikeinterface/preprocessing/filter.py +++ b/src/spikeinterface/preprocessing/filter.py @@ -194,34 +194,42 @@ def get_traces(self, start_frame, end_frame, channel_indices): from scipy.signal import sosfiltfilt, filtfilt, sosfilt, lfilter - if self.direction == "forward-backward": - if self.filter_mode == "sos": - filtered_traces = sosfiltfilt(self.coeff, traces_chunk, axis=0) - elif self.filter_mode == "ba": - b, a = self.coeff - filtered_traces = filtfilt(b, a, traces_chunk, axis=0) - else: - if self.direction == "backward": - traces_chunk = np.flip(traces_chunk, axis=0) + # Channels are filtered independently, so filtering one channel at a time gives the same result with a + # much lower peak memory (scipy makes several float copies of its input) and no loss in speed + num_samples = traces_chunk.shape[0] - left_margin - right_margin + num_channels = traces_chunk.shape[1] + filtered_traces = np.empty((num_samples, num_channels), dtype=self.dtype) + + for channel in range(num_channels): + block = traces_chunk[:, channel : channel + 1] + + if self.direction == "forward-backward": + if self.filter_mode == "sos": + filtered_block = sosfiltfilt(self.coeff, block, axis=0) + elif self.filter_mode == "ba": + b, a = self.coeff + filtered_block = filtfilt(b, a, block, axis=0) + else: + if self.direction == "backward": + block = np.flip(block, axis=0) - if self.filter_mode == "sos": - filtered_traces = sosfilt(self.coeff, traces_chunk, axis=0) - elif self.filter_mode == "ba": - b, a = self.coeff - filtered_traces = lfilter(b, a, traces_chunk, axis=0) + if self.filter_mode == "sos": + filtered_block = sosfilt(self.coeff, block, axis=0) + elif self.filter_mode == "ba": + b, a = self.coeff + filtered_block = lfilter(b, a, block, axis=0) - if self.direction == "backward": - filtered_traces = np.flip(filtered_traces, axis=0) + if self.direction == "backward": + filtered_block = np.flip(filtered_block, axis=0) - if right_margin > 0: - filtered_traces = filtered_traces[left_margin:-right_margin, :] - else: - filtered_traces = filtered_traces[left_margin:, :] + filtered_block = filtered_block[left_margin : filtered_block.shape[0] - right_margin, :] + + if np.issubdtype(self.dtype, np.integer): + filtered_block = filtered_block.round() - if np.issubdtype(self.dtype, np.integer): - filtered_traces = filtered_traces.round() + filtered_traces[:, channel : channel + 1] = filtered_block - return filtered_traces.astype(self.dtype) + return filtered_traces class BandpassFilterRecording(FilterRecording): From 22e1a2a2512a36c95f17d13c025c8efba72937e8 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 7 Oct 2026 17:58:11 -0600 Subject: [PATCH 2/8] Pin git-annex to 10.20260901 in the Linux test job --- .github/workflows/all-tests.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/all-tests.yml b/.github/workflows/all-tests.yml index 033ee7492c..ea4b932fa5 100644 --- a/.github/workflows/all-tests.yml +++ b/.github/workflows/all-tests.yml @@ -129,7 +129,8 @@ jobs: run: | uv pip install --system datalad-installer if [ ${{ runner.os }} = 'Linux' ]; then - datalad-installer --sudo ok git-annex --method datalad/packages + # TODO: remove the pin once datalad uploads the git-annex 10.20261006 package (its latest-version points to a missing file) + datalad-installer --sudo ok git-annex=10.20260901 --method datalad/packages elif [ ${{ runner.os }} = 'macOS' ]; then datalad-installer --sudo ok git-annex --method brew elif [ ${{ runner.os }} = 'Windows' ]; then From fa9fcfccdcd1cab7880e129cd83f1c919fcbb586 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Wed, 7 Oct 2026 18:06:36 -0600 Subject: [PATCH 3/8] rename --- src/spikeinterface/preprocessing/filter.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/spikeinterface/preprocessing/filter.py b/src/spikeinterface/preprocessing/filter.py index e81a158fb6..234c5a7362 100644 --- a/src/spikeinterface/preprocessing/filter.py +++ b/src/spikeinterface/preprocessing/filter.py @@ -200,8 +200,8 @@ def get_traces(self, start_frame, end_frame, channel_indices): num_channels = traces_chunk.shape[1] filtered_traces = np.empty((num_samples, num_channels), dtype=self.dtype) - for channel in range(num_channels): - block = traces_chunk[:, channel : channel + 1] + for channel_index in range(num_channels): + block = traces_chunk[:, channel_index : channel_index + 1] if self.direction == "forward-backward": if self.filter_mode == "sos": @@ -227,7 +227,7 @@ def get_traces(self, start_frame, end_frame, channel_indices): if np.issubdtype(self.dtype, np.integer): filtered_block = filtered_block.round() - filtered_traces[:, channel : channel + 1] = filtered_block + filtered_traces[:, channel_index : channel_index + 1] = filtered_block return filtered_traces From 8642d4d7d81ceb76067dea7298b7290499e25dce Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 8 Oct 2026 16:07:38 +0200 Subject: [PATCH 4/8] fix: remove unused slice Co-authored-by: Alessio Buccino --- src/spikeinterface/preprocessing/filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/preprocessing/filter.py b/src/spikeinterface/preprocessing/filter.py index 234c5a7362..977b28d964 100644 --- a/src/spikeinterface/preprocessing/filter.py +++ b/src/spikeinterface/preprocessing/filter.py @@ -201,7 +201,7 @@ def get_traces(self, start_frame, end_frame, channel_indices): filtered_traces = np.empty((num_samples, num_channels), dtype=self.dtype) for channel_index in range(num_channels): - block = traces_chunk[:, channel_index : channel_index + 1] + block = traces_chunk[:, channel_index] if self.direction == "forward-backward": if self.filter_mode == "sos": @@ -227,7 +227,7 @@ def get_traces(self, start_frame, end_frame, channel_indices): if np.issubdtype(self.dtype, np.integer): filtered_block = filtered_block.round() - filtered_traces[:, channel_index : channel_index + 1] = filtered_block + filtered_traces[:, channel_index] = filtered_block return filtered_traces From 625dbc03c14972ab08d0b271edbe4085b2553d4d Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 8 Oct 2026 16:15:42 +0200 Subject: [PATCH 5/8] Revert "fix: remove unused slice" This reverts commit 8642d4d7d81ceb76067dea7298b7290499e25dce. --- src/spikeinterface/preprocessing/filter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/spikeinterface/preprocessing/filter.py b/src/spikeinterface/preprocessing/filter.py index 977b28d964..234c5a7362 100644 --- a/src/spikeinterface/preprocessing/filter.py +++ b/src/spikeinterface/preprocessing/filter.py @@ -201,7 +201,7 @@ def get_traces(self, start_frame, end_frame, channel_indices): filtered_traces = np.empty((num_samples, num_channels), dtype=self.dtype) for channel_index in range(num_channels): - block = traces_chunk[:, channel_index] + block = traces_chunk[:, channel_index : channel_index + 1] if self.direction == "forward-backward": if self.filter_mode == "sos": @@ -227,7 +227,7 @@ def get_traces(self, start_frame, end_frame, channel_indices): if np.issubdtype(self.dtype, np.integer): filtered_block = filtered_block.round() - filtered_traces[:, channel_index] = filtered_block + filtered_traces[:, channel_index : channel_index + 1] = filtered_block return filtered_traces From edf5334c9d37458d9cff2178ef8e11f3b417a6be Mon Sep 17 00:00:00 2001 From: Alessio Buccino Date: Thu, 8 Oct 2026 16:33:37 +0200 Subject: [PATCH 6/8] test: add missing require zarr write --- src/spikeinterface/core/tests/test_unitsselectionsorting.py | 1 + src/spikeinterface/core/tests/test_zarrextractors.py | 1 + 2 files changed, 2 insertions(+) diff --git a/src/spikeinterface/core/tests/test_unitsselectionsorting.py b/src/spikeinterface/core/tests/test_unitsselectionsorting.py index 0b9cf6c813..fe9f81c075 100644 --- a/src/spikeinterface/core/tests/test_unitsselectionsorting.py +++ b/src/spikeinterface/core/tests/test_unitsselectionsorting.py @@ -248,6 +248,7 @@ def test_non_identity_selection_does_not_share(unit_ids): assert len(parent._cached_lexsorted_spike_vector) == 0 +@pytest.mark.requires_zarr_write def test_identity_selection_keeps_lazy_zarr_vector(tmp_path): """A lazy parent spike vector should stay lazy through an identity selection.""" from spikeinterface.core.zarrextractors import ZarrSortingExtractor, ZarrSpikeVector diff --git a/src/spikeinterface/core/tests/test_zarrextractors.py b/src/spikeinterface/core/tests/test_zarrextractors.py index e1b315e592..2b388839d1 100644 --- a/src/spikeinterface/core/tests/test_zarrextractors.py +++ b/src/spikeinterface/core/tests/test_zarrextractors.py @@ -102,6 +102,7 @@ def test_ZarrSampleIndexSearch(tmp_path): np.testing.assert_array_equal(search.searchsorted([5], 10, 10), [0]) +@pytest.mark.requires_zarr_write def test_ZarrSortingExtractor_lazy_search(tmp_path): sorting = generate_sorting(num_units=10, durations=[5.0, 3.0, 4.0], firing_rates=40.0, seed=0) folder = tmp_path / "sorting.zarr" From 554c125f9e712aae2991e2f1c8a8cf769abce305 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Mon, 12 Oct 2026 12:51:07 -0600 Subject: [PATCH 7/8] Filter traces in internal banks of 16 channels --- src/spikeinterface/preprocessing/filter.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/spikeinterface/preprocessing/filter.py b/src/spikeinterface/preprocessing/filter.py index 234c5a7362..9956951032 100644 --- a/src/spikeinterface/preprocessing/filter.py +++ b/src/spikeinterface/preprocessing/filter.py @@ -194,14 +194,16 @@ def get_traces(self, start_frame, end_frame, channel_indices): from scipy.signal import sosfiltfilt, filtfilt, sosfilt, lfilter - # Channels are filtered independently, so filtering one channel at a time gives the same result with a - # much lower peak memory (scipy makes several float copies of its input) and no loss in speed + # Channels are filtered independently, so filtering in banks gives the same result with a + # much lower peak memory (scipy makes several float copies of its input) num_samples = traces_chunk.shape[0] - left_margin - right_margin num_channels = traces_chunk.shape[1] + num_channels_per_bank = 16 filtered_traces = np.empty((num_samples, num_channels), dtype=self.dtype) - for channel_index in range(num_channels): - block = traces_chunk[:, channel_index : channel_index + 1] + for channel_index in range(0, num_channels, num_channels_per_bank): + channel_slice = slice(channel_index, channel_index + num_channels_per_bank) + block = traces_chunk[:, channel_slice] if self.direction == "forward-backward": if self.filter_mode == "sos": @@ -227,7 +229,7 @@ def get_traces(self, start_frame, end_frame, channel_indices): if np.issubdtype(self.dtype, np.integer): filtered_block = filtered_block.round() - filtered_traces[:, channel_index : channel_index + 1] = filtered_block + filtered_traces[:, channel_slice] = filtered_block return filtered_traces From 869bcd6cc656666c4a188bc4c689ade00eb96493 Mon Sep 17 00:00:00 2001 From: Heberto Mayorquin Date: Tue, 20 Oct 2026 08:10:39 -0600 Subject: [PATCH 8/8] Move the channel block size to a module constant --- src/spikeinterface/preprocessing/filter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/spikeinterface/preprocessing/filter.py b/src/spikeinterface/preprocessing/filter.py index 9956951032..3f4a7aae14 100644 --- a/src/spikeinterface/preprocessing/filter.py +++ b/src/spikeinterface/preprocessing/filter.py @@ -14,6 +14,7 @@ HIGHPASS_ERROR_THRESHOLD_HZ = 100 MARGIN_TO_CHUNK_PERCENT_WARNING = 0.2 # 20% +NUM_CHANNELS_PER_BLOCK = 16 _common_filter_docs = """**filter_kwargs : dict @@ -194,15 +195,14 @@ def get_traces(self, start_frame, end_frame, channel_indices): from scipy.signal import sosfiltfilt, filtfilt, sosfilt, lfilter - # Channels are filtered independently, so filtering in banks gives the same result with a + # Channels are filtered independently, so filtering in blocks gives the same result with a # much lower peak memory (scipy makes several float copies of its input) num_samples = traces_chunk.shape[0] - left_margin - right_margin num_channels = traces_chunk.shape[1] - num_channels_per_bank = 16 filtered_traces = np.empty((num_samples, num_channels), dtype=self.dtype) - for channel_index in range(0, num_channels, num_channels_per_bank): - channel_slice = slice(channel_index, channel_index + num_channels_per_bank) + for channel_index in range(0, num_channels, NUM_CHANNELS_PER_BLOCK): + channel_slice = slice(channel_index, channel_index + NUM_CHANNELS_PER_BLOCK) block = traces_chunk[:, channel_slice] if self.direction == "forward-backward":