diff --git a/solrorbit/worker_coordinator/runner.py b/solrorbit/worker_coordinator/runner.py index 73d65605..74162a16 100644 --- a/solrorbit/worker_coordinator/runner.py +++ b/solrorbit/worker_coordinator/runner.py @@ -1151,18 +1151,18 @@ async def __call__(self, client, params): for doc in doc_stream: batch.append(doc) if len(batch) >= batch_size: + total_docs += len(batch) try: await _run_in_executor(sc.add, collection, batch, commit=False, commitWithin=1000) - total_docs += len(batch) except pysolr.SolrError as exc: logging.getLogger(__name__).error("Bulk index error on batch: %s", exc) errors += len(batch) batch = [] if batch: + total_docs += len(batch) try: await _run_in_executor(sc.add, collection, batch, commit=False, commitWithin=1000) - total_docs += len(batch) except pysolr.SolrError as exc: logging.getLogger(__name__).error("Bulk index error on final batch: %s", exc) errors += len(batch) @@ -1171,6 +1171,7 @@ async def __call__(self, client, params): await _run_in_executor(sc.commit, collection) elapsed = time.perf_counter() - start + # total_docs counts every document sent, so this is the number actually indexed. weight = total_docs - errors return { diff --git a/tests/unit/solr/test_runner.py b/tests/unit/solr/test_runner.py index 352ab808..e9341509 100644 --- a/tests/unit/solr/test_runner.py +++ b/tests/unit/solr/test_runner.py @@ -202,6 +202,41 @@ def test_bulk_index_reports_errors(self): self.assertFalse(result["success"]) self.assertGreater(result["error-count"], 0) + def test_partial_failure_counts_failed_docs_once(self): + """A failed batch is reported as errors, not also subtracted from the indexed docs.""" + import pysolr + mock_sc = MagicMock() + mock_sc.add.side_effect = [None, pysolr.SolrError("Indexing error")] + + lines = [ + '{"index": {"_id": "1"}}', + '{"title": "doc"}', + '{"index": {"_id": "2"}}', + '{"title": "doc"}', + ] + params = {**self._params(lines), "bulk-size": 1} + result = _run(SolrBulkIndex()(mock_sc, params)) + + self.assertEqual(1, result["weight"]) + self.assertEqual(2, result["bulk-size"]) + self.assertEqual(1, result["error-count"]) + self.assertFalse(result["success"]) + + def test_failed_bulk_has_zero_weight(self): + import pysolr + mock_sc = MagicMock() + mock_sc.add.side_effect = pysolr.SolrError("Indexing error") + + lines = [ + '{"index": {"_id": "1"}}', + '{"title": "doc"}', + ] + result = _run(SolrBulkIndex()(mock_sc, self._params(lines))) + + self.assertEqual(0, result["weight"]) + self.assertEqual(1, result["bulk-size"]) + self.assertEqual(1, result["error-count"]) + def test_simple_ndjson_format(self): """Test simple NDJSON (one doc per line, no action lines).""" mock_sc = MagicMock()