diff --git a/CHANGELOG.md b/CHANGELOG.md index 99aa46d..92f41b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,13 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +* Make `Hashdown` downloads (used by `example_file`) more robust: retry + network errors and HTTP 5xx responses up to three times with backoff, time + out stalled connections, and write each file under a temporary name so an + interrupted download never leaves a partial file in the cache. + ## [1.0.0] - 2026-11-13 * Require Python 3.11.5 or later and refresh stable runtime and development diff --git a/pysnptools/test.py b/pysnptools/test.py index f58e15e..8d5d24d 100644 --- a/pysnptools/test.py +++ b/pysnptools/test.py @@ -30,6 +30,7 @@ from pysnptools.util.intrangeset import TestIntRangeSet from pysnptools.util.test import TestUtilTools from pysnptools.util.filecache.test import TestFileCache +from pysnptools.util.filecache.test_hashdown_retry import TestHashdownRetry from pysnptools.distreader.test import TestDistReaders import unittest @@ -1571,6 +1572,7 @@ def getTestSuite(): test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestDistReaders)) test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestDistributedBed)) test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestFileCache)) + test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestHashdownRetry)) test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestUtilTools)) test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestIntRangeSet)) test_suite.addTests(unittest.TestLoader().loadTestsFromTestCase(TestSnpDocStrings)) diff --git a/pysnptools/util/filecache/hashdown.py b/pysnptools/util/filecache/hashdown.py index d14b15d..e4f03eb 100644 --- a/pysnptools/util/filecache/hashdown.py +++ b/pysnptools/util/filecache/hashdown.py @@ -3,8 +3,11 @@ import hashlib import fnmatch -from urllib.request import urlopen, HTTPError +from urllib.request import urlopen +from urllib.error import HTTPError, URLError +import http.client import logging +import time from contextlib import contextmanager from pysnptools.util.filecache import FileCache import tempfile @@ -104,9 +107,19 @@ def name(self): return result + # A download is tried this many times in all before its error is raised. + _DOWNLOAD_ATTEMPTS = 4 + # Seconds a connection may wait for data before the attempt is retried. + _DOWNLOAD_TIMEOUT = 60 + @staticmethod def _get_large_file(url, file, trust_local_files, length=16 * 1024): - """https://stackoverflow.com/questions/1517616/stream-large-binary-files-with-urllib2-to-file""" + """Download url to file, retrying transient network and server errors. + + Returns False if the server reports 404. The file is written to a + temporary name and renamed into place only once it is complete, so an + interrupted download never leaves a partial file behind. + """ import pysnptools.util as pstutil # put here to avoid recursive nesting logging.info("Downloading'{0}'".format(url)) @@ -114,17 +127,43 @@ def _get_large_file(url, file, trust_local_files, length=16 * 1024): logging.info("Trusting local file'{0}'".format(file)) return True - try: - req = urlopen(url) - except HTTPError as e: - if e.code == 404: - return False - raise - pstutil.create_directory_if_necessary(file) - with open(file, "wb") as fp: - shutil.copyfileobj(req, fp, length) - return True + for attempt in range(1, Hashdown._DOWNLOAD_ATTEMPTS + 1): + try: + Hashdown._download_once(url, file, length) + return True + except HTTPError as e: + if e.code == 404: + return False + if e.code < 500 or attempt == Hashdown._DOWNLOAD_ATTEMPTS: + raise + error = e + except (URLError, TimeoutError, ConnectionError, http.client.HTTPException) as e: + if attempt == Hashdown._DOWNLOAD_ATTEMPTS: + raise + error = e + wait = 2 ** (attempt - 1) + logging.warning( + "Download of '{0}' failed ({1}); retrying in {2}s".format(url, error, wait) + ) + time.sleep(wait) + + @staticmethod + def _download_once(url, file, length): + with urlopen(url, timeout=Hashdown._DOWNLOAD_TIMEOUT) as req: + fd, temp_file = tempfile.mkstemp( + dir=os.path.dirname(file) or ".", + prefix=os.path.basename(file) + ".", + suffix=".part", + ) + try: + with os.fdopen(fd, "wb") as fp: + shutil.copyfileobj(req, fp, length) + os.replace(temp_file, file) + except BaseException: + if os.path.exists(temp_file): + os.remove(temp_file) + raise @staticmethod def _get_hash(filename): diff --git a/pysnptools/util/filecache/test_hashdown_retry.py b/pysnptools/util/filecache/test_hashdown_retry.py new file mode 100644 index 0000000..bd23de8 --- /dev/null +++ b/pysnptools/util/filecache/test_hashdown_retry.py @@ -0,0 +1,116 @@ +"""Tests for Hashdown download retries and atomic writes, using a fake urlopen.""" + +import io +import os +import tempfile +import unittest +from unittest import mock +from urllib.error import HTTPError, URLError + +from pysnptools.util.filecache import hashdown +from pysnptools.util.filecache.hashdown import Hashdown + + +class _Body(io.BytesIO): + """A response body; optionally fails partway through reading.""" + + def __init__(self, data, fail_after=None): + super().__init__(data) + self._fail_after = fail_after + + def read(self, size=-1): + if self._fail_after is not None and self.tell() >= self._fail_after: + raise ConnectionResetError("simulated reset mid-download") + return super().read(size if self._fail_after is None else min(size, 4)) + + +def _http_error(code): + return HTTPError("https://example.invalid/f", code, "simulated", {}, None) + + +class TestHashdownRetry(unittest.TestCase): + def setUp(self): + self.dir = tempfile.mkdtemp() + self.file = os.path.join(self.dir, "sub", "f.txt") + sleep = mock.patch.object(hashdown.time, "sleep") + self.sleep = sleep.start() + self.addCleanup(sleep.stop) + + def _get(self, *responses): + calls = [] + + def fake_urlopen(url, timeout=None): + calls.append(timeout) + result = responses[len(calls) - 1] + if isinstance(result, BaseException): + raise result + return result + + with mock.patch.object(hashdown, "urlopen", fake_urlopen): + result = Hashdown._get_large_file("https://example.invalid/f", self.file, False) + return result, calls + + def _leftovers(self): + return [n for n in os.listdir(os.path.dirname(self.file)) if n.endswith(".part")] + + def test_retries_transient_network_error(self): + result, calls = self._get(URLError("simulated DNS failure"), _Body(b"hello")) + self.assertTrue(result) + self.assertEqual(len(calls), 2) + self.assertEqual(calls[0], Hashdown._DOWNLOAD_TIMEOUT) + with open(self.file, "rb") as f: + self.assertEqual(f.read(), b"hello") + self.sleep.assert_called_once_with(1) + + def test_retries_server_error(self): + result, calls = self._get(_http_error(503), _http_error(502), _Body(b"ok")) + self.assertTrue(result) + self.assertEqual(len(calls), 3) + self.assertEqual([c.args[0] for c in self.sleep.call_args_list], [1, 2]) + + def test_404_returns_false_without_retry(self): + result, calls = self._get(_http_error(404)) + self.assertFalse(result) + self.assertEqual(len(calls), 1) + self.sleep.assert_not_called() + + def test_client_error_is_not_retried(self): + with self.assertRaises(HTTPError): + self._get(_http_error(403)) + self.sleep.assert_not_called() + + def test_gives_up_after_all_attempts(self): + errors = [URLError("down")] * Hashdown._DOWNLOAD_ATTEMPTS + with self.assertRaises(URLError): + self._get(*errors) + self.assertEqual(self.sleep.call_count, Hashdown._DOWNLOAD_ATTEMPTS - 1) + + def test_interrupted_download_leaves_no_partial_file(self): + os.makedirs(os.path.dirname(self.file), exist_ok=True) + with open(self.file, "wb") as f: + f.write(b"old complete file") + result, calls = self._get(_Body(b"0123456789", fail_after=4), _Body(b"new")) + self.assertTrue(result) + self.assertEqual(len(calls), 2) + with open(self.file, "rb") as f: + self.assertEqual(f.read(), b"new") + self.assertEqual(self._leftovers(), []) + + def test_failed_download_keeps_previous_file(self): + os.makedirs(os.path.dirname(self.file), exist_ok=True) + with open(self.file, "wb") as f: + f.write(b"old") + errors = [_Body(b"0123456789", fail_after=4) for _ in range(Hashdown._DOWNLOAD_ATTEMPTS)] + with self.assertRaises(ConnectionResetError): + self._get(*errors) + with open(self.file, "rb") as f: + self.assertEqual(f.read(), b"old") + self.assertEqual(self._leftovers(), []) + + +def getTestSuite(): + return unittest.TestLoader().loadTestsFromTestCase(TestHashdownRetry) + + +if __name__ == "__main__": + unittest.main()