Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
29 changes: 25 additions & 4 deletions pysus/api/dadosgov/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -304,6 +304,26 @@ async def download(
return output


def _remote_size(headers: Any) -> int:
"""Extract the total resource size from a HEAD or ranged-GET response.

A ``Range: bytes=0-0`` request is answered with ``206 Partial Content``,
where ``Content-Length`` describes only the returned slice (1 byte) and
the full size is reported by ``Content-Range: bytes 0-0/<total>``.
Reading ``Content-Length`` unconditionally therefore yields ``1`` for
every resource on servers that do not allow HEAD.
"""
content_range = headers.get("Content-Range")
if content_range and "/" in content_range:
# bytes 0-0/12345 (or */12345 for an unsatisfied range)
total = content_range.rsplit("/", 1)[-1].strip()
if total.isdigit():
return int(total)

size = headers.get("Content-Length")
return int(size) if size else 0


class Recurso(BaseModel):
"""A single resource (file) within a dataset on dados.gov.br."""

Expand All @@ -319,8 +339,10 @@ class Recurso(BaseModel):
async def get_size(self) -> int:
"""Retrieve the file size from the remote server.

Makes a HEAD request (falling back to GET with a Range header)
to determine the Content-Length of the resource.
Makes a HEAD request, falling back to GET with a ``Range: bytes=0-0``
header when the server rejects HEAD. The fallback answers with a 206
whose ``Content-Length`` is the length of the single requested byte
(i.e. ``1``); the real total lives in ``Content-Range``.

Returns
-------
Expand All @@ -336,8 +358,7 @@ async def get_size(self) -> int:
headers={"Range": "bytes=0-0"},
)

size = response.headers.get("Content-Length")
return int(size) if size else 0
return _remote_size(response.headers)


class ConjuntoDados(BaseModel):
Expand Down
15 changes: 8 additions & 7 deletions pysus/api/dadosgov/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
from pysus.api.models import BaseRemoteDataset, BaseRemoteFile, BaseRemoteGroup
from pysus.api.types import State

from .client import ConjuntoDados, DadosGov, Recurso
from .client import ConjuntoDados, DadosGov, Recurso, _remote_size
from .metadata import (
DadosGovDatasetExtractor,
DadosGovFileExtractor,
Expand Down Expand Up @@ -195,9 +195,9 @@ async def fetch_metadata(self) -> None:
str(self.path), headers={"Range": "bytes=0-0"}
)

size_str = response.headers.get("Content-Length")
if size_str:
self.record.api_size = int(size_str)
remote_size = _remote_size(response.headers)
if remote_size > 0:
self.record.api_size = remote_size

last_mod_str = response.headers.get("Last-Modified")
if last_mod_str:
Expand All @@ -221,8 +221,9 @@ async def _download(
async def fetch_size(self) -> int:
"""Fetch the remote file size and update the local record.

Makes a HEAD request (falling back to GET with a Range header)
to determine the Content-Length.
Makes a HEAD request, falling back to GET with a ``Range: bytes=0-0``
header when the server rejects HEAD. In the ranged response the
total size comes from ``Content-Range``, not ``Content-Length``.

Returns
-------
Expand All @@ -241,7 +242,7 @@ async def fetch_size(self) -> int:
str(self.path), headers={"Range": "bytes=0-0"}
)

remote_size = int(response.headers.get("Content-Length", 0))
remote_size = _remote_size(response.headers)

if remote_size > 0:
self.record.api_size = remote_size
Expand Down
89 changes: 89 additions & 0 deletions pysus/tests/api/dadosgov/test_client.py
Original file line number Diff line number Diff line change
Expand Up @@ -208,6 +208,95 @@ async def test_get_size_no_content_length(self):

assert size == 0

@pytest.mark.asyncio
async def test_get_size_range_fallback_uses_content_range(self):
"""A 206 reports the slice length, not the total, in Content-Length.

``Range: bytes=0-0`` makes the server answer with a single byte, so
``Content-Length`` is 1. The real size is only in ``Content-Range``.
"""
r = Recurso(
id="r7",
title="Test",
url="https://example.com/big.csv",
api_size=0,
)
head_response = MagicMock()
head_response.status_code = 405

get_response = MagicMock()
get_response.status_code = 206
get_response.headers = {
"Content-Length": "1",
"Content-Range": "bytes 0-0/987654321",
}

mock_client = AsyncMock()
mock_client.head.return_value = head_response
mock_client.get.return_value = get_response
mock_client.__aenter__.return_value = mock_client

with patch("httpx.AsyncClient", return_value=mock_client):
size = await r.get_size()

assert size == 987654321

@pytest.mark.asyncio
async def test_get_size_head_with_content_range_uses_total(self):
r = Recurso(
id="r8",
title="Test",
url="https://example.com/big.csv",
api_size=0,
)
mock_response = MagicMock()
mock_response.status_code = 200
mock_response.headers = {
"Content-Length": "5000",
"Content-Range": "bytes 0-4999/12345",
}

mock_client = AsyncMock()
mock_client.head.return_value = mock_response
mock_client.__aenter__.return_value = mock_client

with patch("httpx.AsyncClient", return_value=mock_client):
size = await r.get_size()

assert size == 12345

@pytest.mark.asyncio
async def test_get_size_malformed_content_range_falls_back(self):
r = Recurso(
id="r9",
title="Test",
url="https://example.com/file.csv",
api_size=0,
)
head_response = MagicMock()
head_response.status_code = 405

get_response = MagicMock()
get_response.headers = {
"Content-Length": "1",
"Content-Range": "bytes 0-0/*",
}

mock_client = AsyncMock()
mock_client.head.return_value = head_response
mock_client.get.return_value = get_response
mock_client.__aenter__.return_value = mock_client

with patch("httpx.AsyncClient", return_value=mock_client):
size = await r.get_size()

assert size == 1

def test_remote_size_without_any_header(self):
from pysus.api.dadosgov.client import _remote_size

assert _remote_size({}) == 0


class TestConjuntoDados:
def test_fields_from_aliases(self):
Expand Down
58 changes: 58 additions & 0 deletions pysus/tests/api/dadosgov/test_models.py
Original file line number Diff line number Diff line change
Expand Up @@ -352,6 +352,64 @@ async def test_exception_is_caught(self):

assert f.record.api_size == 0

@pytest.mark.asyncio
async def test_fetch_metadata_range_fallback_uses_content_range(self):
"""A ranged GET answers 206 with a 1-byte Content-Length.

Recording that as the file size makes the catalog store ``1`` for
every remote file.
"""
recurso = make_recurso(tamanho=0)
ds = MockDataset(client=DadosGov())
f = File(record=recurso, dataset=ds, path="http://example.com/big.csv")

head_response = MagicMock()
head_response.status_code = 405

get_response = MagicMock()
get_response.status_code = 206
get_response.headers = {
"Content-Length": "1",
"Content-Range": "bytes 0-0/42424242",
}

mock_client = AsyncMock()
mock_client.head.return_value = head_response
mock_client.get.return_value = get_response
mock_client.__aenter__.return_value = mock_client

with patch("httpx.AsyncClient", return_value=mock_client):
await f.fetch_metadata()

assert f.record.api_size == 42424242

@pytest.mark.asyncio
async def test_fetch_size_range_fallback_uses_content_range(self):
recurso = make_recurso(tamanho=0)
ds = MockDataset(client=DadosGov())
f = File(record=recurso, dataset=ds, path="http://example.com/big.csv")

head_response = MagicMock()
head_response.status_code = 405

get_response = MagicMock()
get_response.status_code = 206
get_response.headers = {
"Content-Length": "1",
"Content-Range": "bytes 0-0/7777777",
}

mock_client = AsyncMock()
mock_client.head.return_value = head_response
mock_client.get.return_value = get_response
mock_client.__aenter__.return_value = mock_client

with patch("httpx.AsyncClient", return_value=mock_client):
size = await f.fetch_size()

assert size == 7777777
assert f.record.api_size == 7777777

@pytest.mark.asyncio
async def test_parse_typeerror_is_caught(self):
recurso = make_recurso(tamanho=0)
Expand Down
Loading