From 78797203ba17593080ff27517c562bda44bbcb4b Mon Sep 17 00:00:00 2001 From: Weijia Wang <9713184+wegank@users.noreply.github.com> Date: Fri, 2 Oct 2026 01:22:57 +0200 Subject: [PATCH] Fix out-of-bounds read in copy_poly_in_matrix for monomials When a Groebner basis element whose leading monomial is hit by the multiplication by x_n is a monomial (length 1), copy_poly_in_matrix compared the exponent at lmb + (len - 2) * nv, i.e. lmb - nv, with the exponent following the leading term, which belongs to the next polynomial or lies past the end of bexp. This was reported by AddressSanitizer as a heap-buffer-overflow in the unit test fglm_build_matrixn_nonradical-radicalshape-31. A monomial has no tail, so its row in the multiplication matrix is zero; since dense_mat is zero-initialized, return early in that case. The same guard is added to copy_poly_in_matrixcol, which contains the same code. Co-Authored-By: Claude Opus 5.5 --- src/msolve/hilbert.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/msolve/hilbert.c b/src/msolve/hilbert.c index d77669df..77dc17a2 100644 --- a/src/msolve/hilbert.c +++ b/src/msolve/hilbert.c @@ -749,6 +749,13 @@ static inline void copy_poly_in_matrix(sp_matfglm_t* matrix, fprintf(ERRSTREAM, "\n"); #endif + /* pos is the length of the polynomial; if pos < 2, it is a monomial: + * it has no tail, hence the corresponding row stays zero + * (dense_mat is zero-initialized) */ + if(pos < 2){ + return; + } + long N = nrows * (matrix->ncols) - (start + 1); if((end-start) == matrix->ncols + 1){ @@ -801,6 +808,13 @@ copy_poly_in_matrixcol(sp_matfglmcol_t* matrix, long nrows, } fprintf(ERRSTREAM, "\n"); #endif + /* pos is the length of the polynomial; if pos < 2, it is a monomial: + * it has no tail, hence the corresponding row stays zero + * (dense_mat is zero-initialized) */ + if(pos < 2){ + return; + } + long N = nrows * (matrix->ncols) - (start + 1); if((end-start) == matrix->ncols + 1){