From d60a3a3ccb6f98d3997fcc6bf647ddfe98c45e39 Mon Sep 17 00:00:00 2001 From: Francois Girinon Date: Sat, 10 Oct 2026 12:35:26 +0200 Subject: [PATCH 1/3] Fix page hash size limit for large PE files pe_page_hash_calc() sized its buffer from the whole file size, so data outside the sections (e.g. an overlay appended by an installer) was counted even though it is not hashed. For a 3.7 GB installer with a 4 KB SectionAlignment the estimate was 31 MB, above the 16 MB limit: "Page hash size exceeds limit" and signing with -ph failed, while the real page hash is a few KB. Compute the number of pages from the section table instead. The bounds of each section are checked before its size is used, and the estimate never exceeds the previous one based on the file size, so a crafted section table cannot cause a larger allocation than before. Raise the sanity limit to 512 MB: with a 4 GB PE file limit and 512 byte pages the page hash cannot be larger. Add optional large file tests (-DLARGE_FILE_TESTS=ON) that sign and verify a 2.6 GB PE with -ph, with the data in the section and as an overlay. --- cmake/CMakeTest.cmake | 39 ++++++++++++++++++++++ osslsigncode.h | 1 + pe.c | 46 ++++++++++++++++++-------- tests/make_large_pe.py | 74 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 146 insertions(+), 14 deletions(-) create mode 100644 tests/make_large_pe.py diff --git a/cmake/CMakeTest.cmake b/cmake/CMakeTest.cmake index 547ba941..aa0431e9 100644 --- a/cmake/CMakeTest.cmake +++ b/cmake/CMakeTest.cmake @@ -15,6 +15,7 @@ if(Python3_FOUND) if(NOT cryptography_error) message(STATUS "Using python3-cryptography version ${cryptography_output}") option(STOP_SERVER "Stop HTTP server after tests" ON) + option(LARGE_FILE_TESTS "Sign 2.6 GB PE files (slow, needs about 8 GB of disk space and a 64-bit build)" OFF) # Remove http proxy configuration that may change behavior unset(ENV{HTTP_PROXY}) @@ -43,6 +44,7 @@ if(Python3_FOUND) "${CMAKE_CURRENT_SOURCE_DIR}/tests/conf" "${CMAKE_CURRENT_SOURCE_DIR}/tests/client_http.py" "${CMAKE_CURRENT_SOURCE_DIR}/tests/make_certificates.py" + "${CMAKE_CURRENT_SOURCE_DIR}/tests/make_large_pe.py" "${CMAKE_CURRENT_SOURCE_DIR}/tests/start_server.py" "${CMAKE_CURRENT_SOURCE_DIR}/tests/exec.py" DESTINATION "${TEST_DIR}/") @@ -420,6 +422,43 @@ if(Python3_FOUND AND NOT cryptography_error) endforeach(ext ${extensions_nocat}) +### Large PE files ### + # Page hash buffer was sized from the file size, so a large file failed even when + # most of it is an overlay, and was limited to 16 MB for large sections. + if(LARGE_FILE_TESTS) + foreach(kind "section" "overlay") + if(kind STREQUAL "overlay") + set(overlay_arg "overlay") + else(kind STREQUAL "overlay") + set(overlay_arg "") + endif(kind STREQUAL "overlay") + add_test(NAME "large_${kind}_create" + COMMAND ${Python3_EXECUTABLE} "${TEST_DIR}/make_large_pe.py" + "${FILES}/large_${kind}.exe" 2600000000 ${overlay_arg}) + add_test(NAME "large_${kind}_sign" + COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "sign" + "-pkcs12" "${CERTS}/cert.p12" + "-readpass" "${CERTS}/password.txt" + "-ph" "-h" "sha256" + "-in" "${FILES}/large_${kind}.exe" + "-out" "${FILES}/large_${kind}_signed.exe") + add_test(NAME "large_${kind}_verify" + COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "verify" + "-CAfile" "${CERTS}/CACert.pem" + "-ignore-timestamp" + "-in" "${FILES}/large_${kind}_signed.exe") + set_tests_properties("large_${kind}_create" PROPERTIES + DEPENDS "start_server" TIMEOUT 300) + set_tests_properties("large_${kind}_sign" PROPERTIES + DEPENDS "large_${kind}_create" TIMEOUT 900) + set_tests_properties("large_${kind}_verify" PROPERTIES + DEPENDS "large_${kind}_sign" TIMEOUT 900) + list(APPEND ALL_TESTS "large_${kind}_create" "large_${kind}_sign" "large_${kind}_verify") + set(OUTPUT_FILES ${OUTPUT_FILES} + "${FILES}/large_${kind}.exe" "${FILES}/large_${kind}_signed.exe") + endforeach(kind "section" "overlay") + endif(LARGE_FILE_TESTS) + if(OPENSSL_VERSION VERSION_GREATER_EQUAL "3.0.0" OR CURL_FOUND) ### Sign with Time-Stamp Authority ### diff --git a/osslsigncode.h b/osslsigncode.h index e218743d..92819eba 100644 --- a/osslsigncode.h +++ b/osslsigncode.h @@ -132,6 +132,7 @@ #define SIZE_64K 65536 /* 2^16 */ #define SIZE_16M 16777216 /* 2^24 */ +#define SIZE_512M 536870912 /* 2^29 */ /* * Macro names: diff --git a/pe.c b/pe.c index e3ff02eb..b9a50583 100644 --- a/pe.c +++ b/pe.c @@ -984,6 +984,7 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype) BIO *bhash = NULL; uint32_t filebound; size_t pphlen_sz, sections_factor; + uint64_t pages; if (rphlen == NULL || ctx == NULL || ctx->options == NULL || ctx->pe_ctx == NULL || ctx->options->indata == NULL) @@ -1044,20 +1045,6 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype) } pphlen = 4 + mdlen; - /* Compute an upper bound for result size and guard overflow */ - pphlen_sz = (size_t)pphlen; - sections_factor = 3 + (size_t)nsections + ((size_t)ctx->pe_ctx->fileend / pagesize); - if (sections_factor > SIZE_MAX / pphlen_sz) { - fprintf(stderr, "Page hash allocation size would overflow\n"); - return NULL; /* FAILED */ - } - phlen = (int)(pphlen_sz * sections_factor); - /* Sanity limit - page hash shouldn't exceed reasonable size (16 MB) */ - if (phlen < 0 || (size_t)phlen > SIZE_16M) { - fprintf(stderr, "Page hash size exceeds limit: %d\n", phlen); - return NULL; /* FAILED */ - } - /* Determine the file boundary for section data validation */ filebound = ctx->pe_ctx->sigpos ? ctx->pe_ctx->sigpos : ctx->pe_ctx->fileend; @@ -1072,6 +1059,37 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype) } sections = (char *)ctx->options->indata + sect_off; + /* Compute an upper bound for result size from the pages of the sections. + * Data outside the sections (e.g. an overlay appended to the file) is not + * hashed, so the file size alone must not be used here */ + for (i = 0, pages = 0; i < (int)nsections; i++) { + rs = GET_UINT32_LE(sections + (size_t)i * 40 + 16); + ro = GET_UINT32_LE(sections + (size_t)i * 40 + 20); + if (rs == 0) + continue; + /* Validate section bounds against file size before using its size */ + if (ro >= filebound || rs > filebound - ro) { + fprintf(stderr, "Section %d has invalid bounds: offset=0x%08X, size=0x%08X, fileend=0x%08X\n", + i, ro, rs, filebound); + return NULL; /* FAILED */ + } + pages += ((uint64_t)rs + pagesize - 1) / pagesize; + } + /* Never more than the pages of the whole file, overlapping sections + * that would need more are rejected while computing the page hash */ + if (pages > (uint64_t)nsections + ctx->pe_ctx->fileend / pagesize) + pages = (uint64_t)nsections + ctx->pe_ctx->fileend / pagesize; + /* header page + last entry (end of the last section) + 1 spare */ + sections_factor = 3 + (size_t)pages; + pphlen_sz = (size_t)pphlen; + /* Sanity limit - a PE file is at most 4 GB, so even with 512 byte pages + * the page hash stays below 512 MB */ + if (sections_factor > SIZE_512M / pphlen_sz) { + fprintf(stderr, "Page hash size exceeds limit: %zu\n", sections_factor * pphlen_sz); + return NULL; /* FAILED */ + } + phlen = (int)(pphlen_sz * sections_factor); + bhash = BIO_new(BIO_f_md()); if (bhash == NULL) return NULL; diff --git a/tests/make_large_pe.py b/tests/make_large_pe.py new file mode 100644 index 00000000..d32ebb38 --- /dev/null +++ b/tests/make_large_pe.py @@ -0,0 +1,74 @@ +#!/usr/bin/python3 +"""Create a minimal valid sparse PE32+ file of a given size (in bytes).""" + +import struct +import sys + +FILE_ALIGNMENT = 0x200 +SECTION_ALIGNMENT = 0x1000 +HEADERS_SIZE = 0x200 + + +def make_pe(path: str, size: int, overlay: bool = False) -> int: + """Write a PE32+ file with one .text section, return its final size. + + The section fills the file, or only 1 MB of it when overlay is set: the rest + is then data appended after the last section, as in large installers.""" + raw = (1 << 20) if overlay else size - HEADERS_SIZE + raw -= raw % FILE_ALIGNMENT + if not overlay: + size = HEADERS_SIZE + raw + + dos = bytearray(64) + dos[0:2] = b"MZ" + struct.pack_into(" None: + """Parse the command line.""" + if len(sys.argv) not in (3, 4): + print("Usage:\n\t{} FILE SIZE [overlay]".format(sys.argv[0]), file=sys.stderr) + sys.exit(1) + make_pe(sys.argv[1], int(sys.argv[2]), len(sys.argv) == 4) + + +if __name__ == "__main__": + main() + + +# pylint: disable=pointless-string-statement +"""Local Variables: + c-basic-offset: 4 + tab-width: 4 + indent-tabs-mode: nil +End: + vim: set ts=4 expandtab: +""" From 00e425e98ab12ffc31dda9b6b3c9560dc0de584f Mon Sep 17 00:00:00 2001 From: Francois Girinon Date: Wed, 14 Oct 2026 19:17:39 +0200 Subject: [PATCH 2/3] Use 64-bit page offset in page hash loop Raising the page hash limit to 512 MB makes the uint32_t overflow of 'l += pagesize' reachable for a section close to 4 GB: l wraps to zero and the section is hashed again until the page hash buffer is full. --- pe.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pe.c b/pe.c index b9a50583..295a2be9 100644 --- a/pe.c +++ b/pe.c @@ -975,7 +975,8 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype) { uint16_t nsections, opthdr_size; uint32_t alignment, pagesize, hdrsize; - uint32_t rs, ro, l, lastpos = 0; + uint32_t rs, ro, lastpos = 0; + uint64_t l; /* 64-bit: l += pagesize wraps to 0 for a section near 4 GB */ int mdlen, pphlen, phlen, i, pi = 1; size_t written, off, sect_off, sect_tbl, need; u_char *res = NULL, *zeroes = NULL; @@ -1189,7 +1190,7 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype) return NULL; /* FAILED */ } - PUT_UINT32_LE(ro + l, res + (size_t)pi * (size_t)pphlen); + PUT_UINT32_LE((uint32_t)(ro + l), res + (size_t)pi * (size_t)pphlen); bhash = BIO_new(BIO_f_md()); if (bhash == NULL) { From e87d27c67aa3c935018f7dc7c44088f7ce068ce6 Mon Sep 17 00:00:00 2001 From: Francois Girinon Date: Wed, 14 Oct 2026 20:57:48 +0200 Subject: [PATCH 3/3] tests: sign a PE file with a section close to 4 GB Covers the 32-bit page offset overflow in pe_page_hash_calc(). The signed file is over 4 GB and cannot be opened again, so it is not verified. --- cmake/CMakeTest.cmake | 35 +++++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 14 deletions(-) diff --git a/cmake/CMakeTest.cmake b/cmake/CMakeTest.cmake index aa0431e9..138096ca 100644 --- a/cmake/CMakeTest.cmake +++ b/cmake/CMakeTest.cmake @@ -15,7 +15,7 @@ if(Python3_FOUND) if(NOT cryptography_error) message(STATUS "Using python3-cryptography version ${cryptography_output}") option(STOP_SERVER "Stop HTTP server after tests" ON) - option(LARGE_FILE_TESTS "Sign 2.6 GB PE files (slow, needs about 8 GB of disk space and a 64-bit build)" OFF) + option(LARGE_FILE_TESTS "Sign 2.6 GB and 4 GB PE files (slow, needs about 16 GB of disk space and a 64-bit build)" OFF) # Remove http proxy configuration that may change behavior unset(ENV{HTTP_PROXY}) @@ -426,15 +426,18 @@ if(Python3_FOUND AND NOT cryptography_error) # Page hash buffer was sized from the file size, so a large file failed even when # most of it is an overlay, and was limited to 16 MB for large sections. if(LARGE_FILE_TESTS) - foreach(kind "section" "overlay") + # "wrap" has a section of 0xFFFFF200 bytes: the page offset overflowed 32 bits + foreach(kind "section" "overlay" "wrap") + set(large_size 2600000000) + set(overlay_arg "") if(kind STREQUAL "overlay") set(overlay_arg "overlay") - else(kind STREQUAL "overlay") - set(overlay_arg "") + elseif(kind STREQUAL "wrap") + set(large_size 4294964224) endif(kind STREQUAL "overlay") add_test(NAME "large_${kind}_create" COMMAND ${Python3_EXECUTABLE} "${TEST_DIR}/make_large_pe.py" - "${FILES}/large_${kind}.exe" 2600000000 ${overlay_arg}) + "${FILES}/large_${kind}.exe" ${large_size} ${overlay_arg}) add_test(NAME "large_${kind}_sign" COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "sign" "-pkcs12" "${CERTS}/cert.p12" @@ -442,21 +445,25 @@ if(Python3_FOUND AND NOT cryptography_error) "-ph" "-h" "sha256" "-in" "${FILES}/large_${kind}.exe" "-out" "${FILES}/large_${kind}_signed.exe") - add_test(NAME "large_${kind}_verify" - COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "verify" - "-CAfile" "${CERTS}/CACert.pem" - "-ignore-timestamp" - "-in" "${FILES}/large_${kind}_signed.exe") set_tests_properties("large_${kind}_create" PROPERTIES DEPENDS "start_server" TIMEOUT 300) set_tests_properties("large_${kind}_sign" PROPERTIES DEPENDS "large_${kind}_create" TIMEOUT 900) - set_tests_properties("large_${kind}_verify" PROPERTIES - DEPENDS "large_${kind}_sign" TIMEOUT 900) - list(APPEND ALL_TESTS "large_${kind}_create" "large_${kind}_sign" "large_${kind}_verify") + list(APPEND ALL_TESTS "large_${kind}_create" "large_${kind}_sign") + # The signed "wrap" file is over 4 GB and cannot be opened again + if(NOT kind STREQUAL "wrap") + add_test(NAME "large_${kind}_verify" + COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "verify" + "-CAfile" "${CERTS}/CACert.pem" + "-ignore-timestamp" + "-in" "${FILES}/large_${kind}_signed.exe") + set_tests_properties("large_${kind}_verify" PROPERTIES + DEPENDS "large_${kind}_sign" TIMEOUT 900) + list(APPEND ALL_TESTS "large_${kind}_verify") + endif(NOT kind STREQUAL "wrap") set(OUTPUT_FILES ${OUTPUT_FILES} "${FILES}/large_${kind}.exe" "${FILES}/large_${kind}_signed.exe") - endforeach(kind "section" "overlay") + endforeach(kind "section" "overlay" "wrap") endif(LARGE_FILE_TESTS) if(OPENSSL_VERSION VERSION_GREATER_EQUAL "3.0.0" OR CURL_FOUND)