From c968fd61c25a2fb4a247e2b20d87b1adea74a305 Mon Sep 17 00:00:00 2001 From: devgtv Date: Tue, 29 Sep 2026 15:35:00 -0300 Subject: [PATCH] fix(dadosgov): read the total size from Content-Range on ranged GETs All three size probes fall back to `GET` with `Range: bytes=0-0` when a server rejects HEAD. That response is a 206 Partial Content: its `Content-Length` describes the single requested byte, so it is always `1`. The real total is only reported by `Content-Range: bytes 0-0/`, which the code never looked at. Consequence: `Recurso.get_size`, `File.fetch_metadata` and `File.fetch_size` all returned 1 for every resource served by a host that rejects HEAD, and the catalog then persisted size=1. Since the sync path compares that value to decide whether to re-download, every subsequent run saw a size change and re-fetched unchanged files indefinitely. Added a shared `_remote_size` helper that prefers the total from Content-Range, falls back to Content-Length when there is no Content-Range (plain HEAD), and handles an unsatisfiable `bytes 0-0/*` by falling back rather than raising. All three call sites use it. --- pysus/api/dadosgov/client.py | 29 ++++++-- pysus/api/dadosgov/models.py | 15 +++-- pysus/tests/api/dadosgov/test_client.py | 89 +++++++++++++++++++++++++ pysus/tests/api/dadosgov/test_models.py | 58 ++++++++++++++++ 4 files changed, 180 insertions(+), 11 deletions(-) diff --git a/pysus/api/dadosgov/client.py b/pysus/api/dadosgov/client.py index c7cf8088..8e66e29d 100644 --- a/pysus/api/dadosgov/client.py +++ b/pysus/api/dadosgov/client.py @@ -304,6 +304,26 @@ async def download( return output +def _remote_size(headers: Any) -> int: + """Extract the total resource size from a HEAD or ranged-GET response. + + A ``Range: bytes=0-0`` request is answered with ``206 Partial Content``, + where ``Content-Length`` describes only the returned slice (1 byte) and + the full size is reported by ``Content-Range: bytes 0-0/``. + Reading ``Content-Length`` unconditionally therefore yields ``1`` for + every resource on servers that do not allow HEAD. + """ + content_range = headers.get("Content-Range") + if content_range and "/" in content_range: + # bytes 0-0/12345 (or */12345 for an unsatisfied range) + total = content_range.rsplit("/", 1)[-1].strip() + if total.isdigit(): + return int(total) + + size = headers.get("Content-Length") + return int(size) if size else 0 + + class Recurso(BaseModel): """A single resource (file) within a dataset on dados.gov.br.""" @@ -319,8 +339,10 @@ class Recurso(BaseModel): async def get_size(self) -> int: """Retrieve the file size from the remote server. - Makes a HEAD request (falling back to GET with a Range header) - to determine the Content-Length of the resource. + Makes a HEAD request, falling back to GET with a ``Range: bytes=0-0`` + header when the server rejects HEAD. The fallback answers with a 206 + whose ``Content-Length`` is the length of the single requested byte + (i.e. ``1``); the real total lives in ``Content-Range``. Returns ------- @@ -336,8 +358,7 @@ async def get_size(self) -> int: headers={"Range": "bytes=0-0"}, ) - size = response.headers.get("Content-Length") - return int(size) if size else 0 + return _remote_size(response.headers) class ConjuntoDados(BaseModel): diff --git a/pysus/api/dadosgov/models.py b/pysus/api/dadosgov/models.py index 7d4bc89f..b75f14df 100644 --- a/pysus/api/dadosgov/models.py +++ b/pysus/api/dadosgov/models.py @@ -15,7 +15,7 @@ from pysus.api.models import BaseRemoteDataset, BaseRemoteFile, BaseRemoteGroup from pysus.api.types import State -from .client import ConjuntoDados, DadosGov, Recurso +from .client import ConjuntoDados, DadosGov, Recurso, _remote_size from .metadata import ( DadosGovDatasetExtractor, DadosGovFileExtractor, @@ -195,9 +195,9 @@ async def fetch_metadata(self) -> None: str(self.path), headers={"Range": "bytes=0-0"} ) - size_str = response.headers.get("Content-Length") - if size_str: - self.record.api_size = int(size_str) + remote_size = _remote_size(response.headers) + if remote_size > 0: + self.record.api_size = remote_size last_mod_str = response.headers.get("Last-Modified") if last_mod_str: @@ -221,8 +221,9 @@ async def _download( async def fetch_size(self) -> int: """Fetch the remote file size and update the local record. - Makes a HEAD request (falling back to GET with a Range header) - to determine the Content-Length. + Makes a HEAD request, falling back to GET with a ``Range: bytes=0-0`` + header when the server rejects HEAD. In the ranged response the + total size comes from ``Content-Range``, not ``Content-Length``. Returns ------- @@ -241,7 +242,7 @@ async def fetch_size(self) -> int: str(self.path), headers={"Range": "bytes=0-0"} ) - remote_size = int(response.headers.get("Content-Length", 0)) + remote_size = _remote_size(response.headers) if remote_size > 0: self.record.api_size = remote_size diff --git a/pysus/tests/api/dadosgov/test_client.py b/pysus/tests/api/dadosgov/test_client.py index b8658182..b24cc42d 100644 --- a/pysus/tests/api/dadosgov/test_client.py +++ b/pysus/tests/api/dadosgov/test_client.py @@ -208,6 +208,95 @@ async def test_get_size_no_content_length(self): assert size == 0 + @pytest.mark.asyncio + async def test_get_size_range_fallback_uses_content_range(self): + """A 206 reports the slice length, not the total, in Content-Length. + + ``Range: bytes=0-0`` makes the server answer with a single byte, so + ``Content-Length`` is 1. The real size is only in ``Content-Range``. + """ + r = Recurso( + id="r7", + title="Test", + url="https://example.com/big.csv", + api_size=0, + ) + head_response = MagicMock() + head_response.status_code = 405 + + get_response = MagicMock() + get_response.status_code = 206 + get_response.headers = { + "Content-Length": "1", + "Content-Range": "bytes 0-0/987654321", + } + + mock_client = AsyncMock() + mock_client.head.return_value = head_response + mock_client.get.return_value = get_response + mock_client.__aenter__.return_value = mock_client + + with patch("httpx.AsyncClient", return_value=mock_client): + size = await r.get_size() + + assert size == 987654321 + + @pytest.mark.asyncio + async def test_get_size_head_with_content_range_uses_total(self): + r = Recurso( + id="r8", + title="Test", + url="https://example.com/big.csv", + api_size=0, + ) + mock_response = MagicMock() + mock_response.status_code = 200 + mock_response.headers = { + "Content-Length": "5000", + "Content-Range": "bytes 0-4999/12345", + } + + mock_client = AsyncMock() + mock_client.head.return_value = mock_response + mock_client.__aenter__.return_value = mock_client + + with patch("httpx.AsyncClient", return_value=mock_client): + size = await r.get_size() + + assert size == 12345 + + @pytest.mark.asyncio + async def test_get_size_malformed_content_range_falls_back(self): + r = Recurso( + id="r9", + title="Test", + url="https://example.com/file.csv", + api_size=0, + ) + head_response = MagicMock() + head_response.status_code = 405 + + get_response = MagicMock() + get_response.headers = { + "Content-Length": "1", + "Content-Range": "bytes 0-0/*", + } + + mock_client = AsyncMock() + mock_client.head.return_value = head_response + mock_client.get.return_value = get_response + mock_client.__aenter__.return_value = mock_client + + with patch("httpx.AsyncClient", return_value=mock_client): + size = await r.get_size() + + assert size == 1 + + def test_remote_size_without_any_header(self): + from pysus.api.dadosgov.client import _remote_size + + assert _remote_size({}) == 0 + class TestConjuntoDados: def test_fields_from_aliases(self): diff --git a/pysus/tests/api/dadosgov/test_models.py b/pysus/tests/api/dadosgov/test_models.py index e09a06b9..f114c8f8 100644 --- a/pysus/tests/api/dadosgov/test_models.py +++ b/pysus/tests/api/dadosgov/test_models.py @@ -352,6 +352,64 @@ async def test_exception_is_caught(self): assert f.record.api_size == 0 + @pytest.mark.asyncio + async def test_fetch_metadata_range_fallback_uses_content_range(self): + """A ranged GET answers 206 with a 1-byte Content-Length. + + Recording that as the file size makes the catalog store ``1`` for + every remote file. + """ + recurso = make_recurso(tamanho=0) + ds = MockDataset(client=DadosGov()) + f = File(record=recurso, dataset=ds, path="http://example.com/big.csv") + + head_response = MagicMock() + head_response.status_code = 405 + + get_response = MagicMock() + get_response.status_code = 206 + get_response.headers = { + "Content-Length": "1", + "Content-Range": "bytes 0-0/42424242", + } + + mock_client = AsyncMock() + mock_client.head.return_value = head_response + mock_client.get.return_value = get_response + mock_client.__aenter__.return_value = mock_client + + with patch("httpx.AsyncClient", return_value=mock_client): + await f.fetch_metadata() + + assert f.record.api_size == 42424242 + + @pytest.mark.asyncio + async def test_fetch_size_range_fallback_uses_content_range(self): + recurso = make_recurso(tamanho=0) + ds = MockDataset(client=DadosGov()) + f = File(record=recurso, dataset=ds, path="http://example.com/big.csv") + + head_response = MagicMock() + head_response.status_code = 405 + + get_response = MagicMock() + get_response.status_code = 206 + get_response.headers = { + "Content-Length": "1", + "Content-Range": "bytes 0-0/7777777", + } + + mock_client = AsyncMock() + mock_client.head.return_value = head_response + mock_client.get.return_value = get_response + mock_client.__aenter__.return_value = mock_client + + with patch("httpx.AsyncClient", return_value=mock_client): + size = await f.fetch_size() + + assert size == 7777777 + assert f.record.api_size == 7777777 + @pytest.mark.asyncio async def test_parse_typeerror_is_caught(self): recurso = make_recurso(tamanho=0)