Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file.
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).

## [Unreleased]

* Make `Hashdown` downloads (used by `example_file`) more robust: retry
network errors and HTTP 5xx responses up to three times with backoff, time
out stalled connections, and write each file under a temporary name so an
interrupted download never leaves a partial file in the cache.

## [1.0.0] - 2026-11-13

* Require Python 3.11.5 or later and refresh stable runtime and development
Expand Down
2 changes: 2 additions & 0 deletions pysnptools/test.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
from pysnptools.util.intrangeset import TestIntRangeSet
from pysnptools.util.test import TestUtilTools
from pysnptools.util.filecache.test import TestFileCache
from pysnptools.util.filecache.test_hashdown_retry import TestHashdownRetry
from pysnptools.distreader.test import TestDistReaders

import unittest
Expand Down Expand Up @@ -1571,6 +1572,7 @@ def getTestSuite():
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestDistReaders))
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestDistributedBed))
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestFileCache))
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestHashdownRetry))
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestUtilTools))
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestIntRangeSet))
test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestSnpDocStrings))
Expand Down
63 changes: 51 additions & 12 deletions pysnptools/util/filecache/hashdown.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,11 @@
import hashlib
import fnmatch

from urllib.request import urlopen, HTTPError
from urllib.request import urlopen
from urllib.error import HTTPError, URLError
import http.client
import logging
import time
from contextlib import contextmanager
from pysnptools.util.filecache import FileCache
import tempfile
Expand Down Expand Up @@ -104,27 +107,63 @@ def name(self):

return result

# A download is tried this many times in all before its error is raised.
_DOWNLOAD_ATTEMPTS = 4
# Seconds a connection may wait for data before the attempt is retried.
_DOWNLOAD_TIMEOUT = 60

@staticmethod
def _get_large_file(url, file, trust_local_files, length=16 * 1024):
"""https://stackoverflow.com/questions/1517616/stream-large-binary-files-with-urllib2-to-file"""
"""Download url to file, retrying transient network and server errors.

Returns False if the server reports 404. The file is written to a
temporary name and renamed into place only once it is complete, so an
interrupted download never leaves a partial file behind.
"""
import pysnptools.util as pstutil # put here to avoid recursive nesting

logging.info("Downloading'{0}'".format(url))
if trust_local_files and os.path.exists(file):
logging.info("Trusting local file'{0}'".format(file))
return True

try:
req = urlopen(url)
except HTTPError as e:
if e.code == 404:
return False
raise

pstutil.create_directory_if_necessary(file)
with open(file, "wb") as fp:
shutil.copyfileobj(req, fp, length)
return True
for attempt in range(1, Hashdown._DOWNLOAD_ATTEMPTS + 1):
try:
Hashdown._download_once(url, file, length)
return True
except HTTPError as e:
if e.code == 404:
return False
if e.code < 500 or attempt == Hashdown._DOWNLOAD_ATTEMPTS:
raise
error = e
except (URLError, TimeoutError, ConnectionError, http.client.HTTPException) as e:
if attempt == Hashdown._DOWNLOAD_ATTEMPTS:
raise
error = e
wait = 2 ** (attempt - 1)
logging.warning(
"Download of '{0}' failed ({1}); retrying in {2}s".format(url, error, wait)
)
time.sleep(wait)

@staticmethod
def _download_once(url, file, length):
with urlopen(url, timeout=Hashdown._DOWNLOAD_TIMEOUT) as req:
fd, temp_file = tempfile.mkstemp(
dir=os.path.dirname(file) or ".",
prefix=os.path.basename(file) + ".",
suffix=".part",
)
try:
with os.fdopen(fd, "wb") as fp:
shutil.copyfileobj(req, fp, length)
os.replace(temp_file, file)
except BaseException:
if os.path.exists(temp_file):
os.remove(temp_file)
raise

@staticmethod
def _get_hash(filename):
Expand Down
116 changes: 116 additions & 0 deletions pysnptools/util/filecache/test_hashdown_retry.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
"""Tests for Hashdown download retries and atomic writes, using a fake urlopen."""

import io
import os
import tempfile
import unittest
from unittest import mock
from urllib.error import HTTPError, URLError

from pysnptools.util.filecache import hashdown
from pysnptools.util.filecache.hashdown import Hashdown


class _Body(io.BytesIO):
"""A response body; optionally fails partway through reading."""

def __init__(self, data, fail_after=None):
super().__init__(data)
self._fail_after = fail_after

def read(self, size=-1):
if self._fail_after is not None and self.tell() >= self._fail_after:
raise ConnectionResetError("simulated reset mid-download")
return super().read(size if self._fail_after is None else min(size, 4))


def _http_error(code):
return HTTPError("https://example.invalid/f", code, "simulated", {}, None)


class TestHashdownRetry(unittest.TestCase):
def setUp(self):
self.dir = tempfile.mkdtemp()
self.file = os.path.join(self.dir, "sub", "f.txt")
sleep = mock.patch.object(hashdown.time, "sleep")
self.sleep = sleep.start()
self.addCleanup(sleep.stop)

def _get(self, *responses):
calls = []

def fake_urlopen(url, timeout=None):
calls.append(timeout)
result = responses[len(calls) - 1]
if isinstance(result, BaseException):
raise result
return result

with mock.patch.object(hashdown, "urlopen", fake_urlopen):
result = Hashdown._get_large_file("https://example.invalid/f", self.file, False)
return result, calls

def _leftovers(self):
return [n for n in os.listdir(os.path.dirname(self.file)) if n.endswith(".part")]

def test_retries_transient_network_error(self):
result, calls = self._get(URLError("simulated DNS failure"), _Body(b"hello"))
self.assertTrue(result)
self.assertEqual(len(calls), 2)
self.assertEqual(calls[0], Hashdown._DOWNLOAD_TIMEOUT)
with open(self.file, "rb") as f:
self.assertEqual(f.read(), b"hello")
self.sleep.assert_called_once_with(1)

def test_retries_server_error(self):
result, calls = self._get(_http_error(503), _http_error(502), _Body(b"ok"))
self.assertTrue(result)
self.assertEqual(len(calls), 3)
self.assertEqual([c.args[0] for c in self.sleep.call_args_list], [1, 2])

def test_404_returns_false_without_retry(self):
result, calls = self._get(_http_error(404))
self.assertFalse(result)
self.assertEqual(len(calls), 1)
self.sleep.assert_not_called()

def test_client_error_is_not_retried(self):
with self.assertRaises(HTTPError):
self._get(_http_error(403))
self.sleep.assert_not_called()

def test_gives_up_after_all_attempts(self):
errors = [URLError("down")] * Hashdown._DOWNLOAD_ATTEMPTS
with self.assertRaises(URLError):
self._get(*errors)
self.assertEqual(self.sleep.call_count, Hashdown._DOWNLOAD_ATTEMPTS - 1)

def test_interrupted_download_leaves_no_partial_file(self):
os.makedirs(os.path.dirname(self.file), exist_ok=True)
with open(self.file, "wb") as f:
f.write(b"old complete file")
result, calls = self._get(_Body(b"0123456789", fail_after=4), _Body(b"new"))
self.assertTrue(result)
self.assertEqual(len(calls), 2)
with open(self.file, "rb") as f:
self.assertEqual(f.read(), b"new")
self.assertEqual(self._leftovers(), [])

def test_failed_download_keeps_previous_file(self):
os.makedirs(os.path.dirname(self.file), exist_ok=True)
with open(self.file, "wb") as f:
f.write(b"old")
errors = [_Body(b"0123456789", fail_after=4) for _ in range(Hashdown._DOWNLOAD_ATTEMPTS)]
with self.assertRaises(ConnectionResetError):
self._get(*errors)
with open(self.file, "rb") as f:
self.assertEqual(f.read(), b"old")
self.assertEqual(self._leftovers(), [])


def getTestSuite():
return unittest.TestLoader().loadTestsFromTestCase(TestHashdownRetry)


if __name__ == "__main__":
unittest.main()
Loading