Fix page hash size limit for large PE files

cmake/CMakeTest.cmake

@@ -15,6 +15,7 @@ if(Python3_FOUND)
     if(NOT cryptography_error)
         message(STATUS "Using python3-cryptography version ${cryptography_output}")
         option(STOP_SERVER "Stop HTTP server after tests" ON)
+        option(LARGE_FILE_TESTS "Sign 2.6 GB and 4 GB PE files (slow, needs about 16 GB of disk space and a 64-bit build)" OFF)
 
         # Remove http proxy configuration that may change behavior
         unset(ENV{HTTP_PROXY})
@@ -43,6 +44,7 @@ if(Python3_FOUND)
             "${CMAKE_CURRENT_SOURCE_DIR}/tests/conf"
             "${CMAKE_CURRENT_SOURCE_DIR}/tests/client_http.py"
             "${CMAKE_CURRENT_SOURCE_DIR}/tests/make_certificates.py"
+            "${CMAKE_CURRENT_SOURCE_DIR}/tests/make_large_pe.py"
             "${CMAKE_CURRENT_SOURCE_DIR}/tests/start_server.py"
             "${CMAKE_CURRENT_SOURCE_DIR}/tests/exec.py"
             DESTINATION "${TEST_DIR}/")
@@ -420,6 +422,50 @@ if(Python3_FOUND AND NOT cryptography_error)
     endforeach(ext ${extensions_nocat})
 
 
+### Large PE files ###
+    # Page hash buffer was sized from the file size, so a large file failed even when
+    # most of it is an overlay, and was limited to 16 MB for large sections.
+    if(LARGE_FILE_TESTS)
+        # "wrap" has a section of 0xFFFFF200 bytes: the page offset overflowed 32 bits
+        foreach(kind "section" "overlay" "wrap")
+            set(large_size 2600000000)
+            set(overlay_arg "")
+            if(kind STREQUAL "overlay")
+                set(overlay_arg "overlay")
+            elseif(kind STREQUAL "wrap")
+                set(large_size 4294964224)
+            endif(kind STREQUAL "overlay")
+            add_test(NAME "large_${kind}_create"
+                COMMAND ${Python3_EXECUTABLE} "${TEST_DIR}/make_large_pe.py"
+                "${FILES}/large_${kind}.exe" ${large_size} ${overlay_arg})
+            add_test(NAME "large_${kind}_sign"
+                COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "sign"
+                "-pkcs12" "${CERTS}/cert.p12"
+                "-readpass" "${CERTS}/password.txt"
+                "-ph" "-h" "sha256"
+                "-in" "${FILES}/large_${kind}.exe"
+                "-out" "${FILES}/large_${kind}_signed.exe")
+            set_tests_properties("large_${kind}_create" PROPERTIES
+                DEPENDS "start_server" TIMEOUT 300)
+            set_tests_properties("large_${kind}_sign" PROPERTIES
+                DEPENDS "large_${kind}_create" TIMEOUT 900)
+            list(APPEND ALL_TESTS "large_${kind}_create" "large_${kind}_sign")
+            # The signed "wrap" file is over 4 GB and cannot be opened again
+            if(NOT kind STREQUAL "wrap")
+                add_test(NAME "large_${kind}_verify"
+                    COMMAND ${Python3_EXECUTABLE} ${EXEC} ${OSSLSIGNCODE} "verify"
+                    "-CAfile" "${CERTS}/CACert.pem"
+                    "-ignore-timestamp"
+                    "-in" "${FILES}/large_${kind}_signed.exe")
+                set_tests_properties("large_${kind}_verify" PROPERTIES
+                    DEPENDS "large_${kind}_sign" TIMEOUT 900)
+                list(APPEND ALL_TESTS "large_${kind}_verify")
+            endif(NOT kind STREQUAL "wrap")
+            set(OUTPUT_FILES ${OUTPUT_FILES}
+                "${FILES}/large_${kind}.exe" "${FILES}/large_${kind}_signed.exe")
+        endforeach(kind "section" "overlay" "wrap")
+    endif(LARGE_FILE_TESTS)
+
     if(OPENSSL_VERSION VERSION_GREATER_EQUAL "3.0.0" OR CURL_FOUND)
 
 ### Sign with Time-Stamp Authority ###

osslsigncode.h

@@ -132,6 +132,7 @@
 
 #define SIZE_64K 65536       /* 2^16 */
 #define SIZE_16M 16777216    /* 2^24 */
+#define SIZE_512M 536870912  /* 2^29 */
 
 /*
  * Macro names:

pe.c

@@ -975,7 +975,8 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype)
 {
     uint16_t nsections, opthdr_size;
     uint32_t alignment, pagesize, hdrsize;
-    uint32_t rs, ro, l, lastpos = 0;
+    uint32_t rs, ro, lastpos = 0;
+    uint64_t l;  /* 64-bit: l += pagesize wraps to 0 for a section near 4 GB */
     int mdlen, pphlen, phlen, i, pi = 1;
     size_t written, off, sect_off, sect_tbl, need;
     u_char *res = NULL, *zeroes = NULL;
@@ -984,6 +985,7 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype)
     BIO *bhash = NULL;
     uint32_t filebound;
     size_t pphlen_sz, sections_factor;
+    uint64_t pages;
 
     if (rphlen == NULL || ctx == NULL || ctx->options == NULL || ctx->pe_ctx == NULL
         || ctx->options->indata == NULL)
@@ -1044,20 +1046,6 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype)
     }
     pphlen = 4 + mdlen;
 
-    /* Compute an upper bound for result size and guard overflow */
-    pphlen_sz = (size_t)pphlen;
-    sections_factor = 3 + (size_t)nsections + ((size_t)ctx->pe_ctx->fileend / pagesize);
-    if (sections_factor > SIZE_MAX / pphlen_sz) {
-        fprintf(stderr, "Page hash allocation size would overflow\n");
-        return NULL; /* FAILED */
-    }
-    phlen = (int)(pphlen_sz * sections_factor);
-    /* Sanity limit - page hash shouldn't exceed reasonable size (16 MB) */
-    if (phlen < 0 || (size_t)phlen > SIZE_16M) {
-        fprintf(stderr, "Page hash size exceeds limit: %d\n", phlen);
-        return NULL; /* FAILED */
-    }
-
     /* Determine the file boundary for section data validation */
     filebound = ctx->pe_ctx->sigpos ? ctx->pe_ctx->sigpos : ctx->pe_ctx->fileend;
 
@@ -1072,6 +1060,37 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype)
     }
     sections = (char *)ctx->options->indata + sect_off;
 
+    /* Compute an upper bound for result size from the pages of the sections.
+     * Data outside the sections (e.g. an overlay appended to the file) is not
+     * hashed, so the file size alone must not be used here */
+    for (i = 0, pages = 0; i < (int)nsections; i++) {
+        rs = GET_UINT32_LE(sections + (size_t)i * 40 + 16);
+        ro = GET_UINT32_LE(sections + (size_t)i * 40 + 20);
+        if (rs == 0)
+            continue;
+        /* Validate section bounds against file size before using its size */
+        if (ro >= filebound || rs > filebound - ro) {
+            fprintf(stderr, "Section %d has invalid bounds: offset=0x%08X, size=0x%08X, fileend=0x%08X\n",
+                    i, ro, rs, filebound);
+            return NULL; /* FAILED */
+        }
+        pages += ((uint64_t)rs + pagesize - 1) / pagesize;
+    }
+    /* Never more than the pages of the whole file, overlapping sections
+     * that would need more are rejected while computing the page hash */
+    if (pages > (uint64_t)nsections + ctx->pe_ctx->fileend / pagesize)
+        pages = (uint64_t)nsections + ctx->pe_ctx->fileend / pagesize;
+    /* header page + last entry (end of the last section) + 1 spare */
+    sections_factor = 3 + (size_t)pages;
+    pphlen_sz = (size_t)pphlen;
+    /* Sanity limit - a PE file is at most 4 GB, so even with 512 byte pages
+     * the page hash stays below 512 MB */
+    if (sections_factor > SIZE_512M / pphlen_sz) {
+        fprintf(stderr, "Page hash size exceeds limit: %zu\n", sections_factor * pphlen_sz);
+        return NULL; /* FAILED */
+    }
+    phlen = (int)(pphlen_sz * sections_factor);
+
     bhash = BIO_new(BIO_f_md());
     if (bhash == NULL)
         return NULL;
@@ -1171,7 +1190,7 @@ static u_char *pe_page_hash_calc(int *rphlen, FILE_FORMAT_CTX *ctx, int phtype)
                 return NULL; /* FAILED */
             }
 
-            PUT_UINT32_LE(ro + l, res + (size_t)pi * (size_t)pphlen);
+            PUT_UINT32_LE((uint32_t)(ro + l), res + (size_t)pi * (size_t)pphlen);
 
             bhash = BIO_new(BIO_f_md());
             if (bhash == NULL) {

tests/make_large_pe.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python3
+"""Create a minimal valid sparse PE32+ file of a given size (in bytes)."""
+
+import struct
+import sys
+
+FILE_ALIGNMENT = 0x200
+SECTION_ALIGNMENT = 0x1000
+HEADERS_SIZE = 0x200
+
+
+def make_pe(path: str, size: int, overlay: bool = False) -> int:
+    """Write a PE32+ file with one .text section, return its final size.
+
+    The section fills the file, or only 1 MB of it when overlay is set: the rest
+    is then data appended after the last section, as in large installers."""
+    raw = (1 << 20) if overlay else size - HEADERS_SIZE
+    raw -= raw % FILE_ALIGNMENT
+    if not overlay:
+        size = HEADERS_SIZE + raw
+
+    dos = bytearray(64)
+    dos[0:2] = b"MZ"
+    struct.pack_into("<I", dos, 60, 64)  # e_lfanew
+    opt_size = 240
+    coff = struct.pack("<HHIIIHH", 0x8664, 1, 0, 0, 0, opt_size, 0x22)
+    opt = bytearray(opt_size)
+    struct.pack_into("<H", opt, 0, 0x20B)
+    struct.pack_into("<I", opt, 16, SECTION_ALIGNMENT)  # entry point
+    struct.pack_into("<Q", opt, 24, 0x140000000)  # image base
+    struct.pack_into("<I", opt, 32, SECTION_ALIGNMENT)
+    struct.pack_into("<I", opt, 36, FILE_ALIGNMENT)
+    struct.pack_into("<H", opt, 40, 6)
+    struct.pack_into("<H", opt, 48, 6)
+    image_size = SECTION_ALIGNMENT + ((raw + SECTION_ALIGNMENT - 1) & ~(SECTION_ALIGNMENT - 1))
+    struct.pack_into("<I", opt, 56, min(image_size, 0xFFFFF000))
+    struct.pack_into("<I", opt, 60, HEADERS_SIZE)
+    struct.pack_into("<H", opt, 68, 3)  # subsystem
+    struct.pack_into("<I", opt, 108, 16)  # NumberOfRvaAndSizes
+    sect = struct.pack("<8sIIIIIIHHI", b".text", raw, SECTION_ALIGNMENT, raw,
+                       HEADERS_SIZE, 0, 0, 0, 0, 0x60000020)
+
+    head = bytes(dos) + b"PE\0\0" + coff + bytes(opt) + sect
+    head += bytes(HEADERS_SIZE - len(head))
+    with open(path, "wb") as file:
+        file.write(head)
+        file.truncate(size)
+        file.seek(HEADERS_SIZE)
+        file.write(b"\x90" * 4096)
+        file.seek(size - 4096)
+        file.write(b"\xCC" * 4096)  # last bytes: section data or overlay
+    return size
+
+
+def main() -> None:
+    """Parse the command line."""
+    if len(sys.argv) not in (3, 4):
+        print("Usage:\n\t{} FILE SIZE [overlay]".format(sys.argv[0]), file=sys.stderr)
+        sys.exit(1)
+    make_pe(sys.argv[1], int(sys.argv[2]), len(sys.argv) == 4)
+
+
+if __name__ == "__main__":
+    main()
+
+
+# pylint: disable=pointless-string-statement
+"""Local Variables:
+    c-basic-offset: 4
+    tab-width: 4
+    indent-tabs-mode: nil
+End:
+    vim: set ts=4 expandtab:
+"""