diff --git a/fesod-sheet/src/main/java/org/apache/fesod/sheet/context/WriteContextImpl.java b/fesod-sheet/src/main/java/org/apache/fesod/sheet/context/WriteContextImpl.java index 17fcc7aadd..5f5e4de825 100644 --- a/fesod-sheet/src/main/java/org/apache/fesod/sheet/context/WriteContextImpl.java +++ b/fesod-sheet/src/main/java/org/apache/fesod/sheet/context/WriteContextImpl.java @@ -27,6 +27,8 @@ import java.io.File; import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.util.List; import java.util.Map; @@ -35,6 +37,7 @@ import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.apache.commons.collections4.MapUtils; +import org.apache.commons.io.IOUtils; import org.apache.fesod.common.util.ListUtils; import org.apache.fesod.common.util.StringUtils; import org.apache.fesod.sheet.enums.HeaderMergeStrategy; @@ -128,20 +131,37 @@ public WriteContextImpl(WriteWorkbook writeWorkbook) { } initCurrentWorkbookHolder(writeWorkbook); - WorkbookWriteHandlerContext workbookWriteHandlerContext = - WriteHandlerUtils.createWorkbookWriteHandlerContext(this); - WriteHandlerUtils.beforeWorkbookCreate(workbookWriteHandlerContext); - try { - WorkBookUtil.createWorkBook(writeWorkbookHolder); - } catch (Exception e) { - throw new ExcelGenerateException("Create workbook failure", e); + try (InputStream template = writeWorkbookHolder.getTempTemplateInputStream()) { + WorkbookWriteHandlerContext workbookWriteHandlerContext = + WriteHandlerUtils.createWorkbookWriteHandlerContext(this); + WriteHandlerUtils.beforeWorkbookCreate(workbookWriteHandlerContext); + try { + WorkBookUtil.createWorkBook(writeWorkbookHolder); + } catch (Exception e) { + throw new ExcelGenerateException("Create workbook failure", e); + } + WriteHandlerUtils.afterWorkbookCreate(workbookWriteHandlerContext); + } catch (IOException e) { + closeOnInitializationFailure(e); + throw new ExcelGenerateException("Close template failure", e); + } catch (Throwable e) { + closeOnInitializationFailure(e); + throw e; } - WriteHandlerUtils.afterWorkbookCreate(workbookWriteHandlerContext); if (log.isDebugEnabled()) { log.debug("Initialization 'WriteContextImpl' complete"); } } + private void closeOnInitializationFailure(Throwable failure) { + IOUtils.closeQuietly(writeWorkbookHolder.getWorkbook(), e -> recordFailure(failure, e)); + if (writeWorkbookHolder.shouldCloseOutputStream()) { + IOUtils.closeQuietly(writeWorkbookHolder.getOutputStream(), e -> recordFailure(failure, e)); + } + clearEncrypt03(); + removeThreadLocalCache(); + } + /** * Initializes the current workbook holder with the provided WriteWorkbook. * @@ -563,7 +583,7 @@ public void finish(boolean onException) { throwable = recordFailure(throwable, t); } try { - if (writeWorkbookHolder.getAutoCloseStream() && writeWorkbookHolder.getOutputStream() != null) { + if (writeWorkbookHolder.shouldCloseOutputStream() && writeWorkbookHolder.getOutputStream() != null) { writeWorkbookHolder.getOutputStream().close(); } } catch (Throwable t) { diff --git a/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/builder/ExcelWriterBuilder.java b/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/builder/ExcelWriterBuilder.java index 56d4a6788c..bef38a7828 100644 --- a/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/builder/ExcelWriterBuilder.java +++ b/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/builder/ExcelWriterBuilder.java @@ -49,7 +49,8 @@ public ExcelWriterBuilder() { } /** - * Default true + * Whether to close caller-provided template input and output streams. Default true. + * Streams opened internally for files are always closed. * * @param autoCloseStream * @return @@ -128,8 +129,8 @@ public ExcelWriterBuilder withBom(Boolean withBom) { } /** - * Template file. - * This file is read into memory, excessive cases can lead to OOM. + * Template stream, read through a bounded buffer when the writer is built. + * The parsed workbook still resides in memory, so large templates can lead to OOM. */ public ExcelWriterBuilder withTemplate(InputStream templateInputStream) { writeWorkbook.setTemplateInputStream(templateInputStream); @@ -137,8 +138,8 @@ public ExcelWriterBuilder withTemplate(InputStream templateInputStream) { } /** - * Template file. - * This file is read into memory, excessive cases can lead to OOM. + * Template file, read through a bounded buffer when the writer is built. + * The parsed workbook still resides in memory, so large templates can lead to OOM. */ public ExcelWriterBuilder withTemplate(File templateFile) { writeWorkbook.setTemplateFile(templateFile); @@ -146,8 +147,8 @@ public ExcelWriterBuilder withTemplate(File templateFile) { } /** - * Template file. - * This file is read into memory, excessive cases can lead to OOM. + * Template file, read through a bounded buffer when the writer is built. + * The parsed workbook still resides in memory, so large templates can lead to OOM. */ public ExcelWriterBuilder withTemplate(String pathName) { return withTemplate(new File(pathName)); diff --git a/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/WriteWorkbook.java b/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/WriteWorkbook.java index 1b96c03b0f..a4028b953d 100644 --- a/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/WriteWorkbook.java +++ b/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/WriteWorkbook.java @@ -73,21 +73,23 @@ public class WriteWorkbook extends WriteBasicParameter { private Boolean withBom; /** - * Template input stream + * Template input stream, read through a bounded buffer when the writer is built. + * The parsed workbook still resides in memory. *
* If 'inputStream' and 'file' all not empty, file first */ private InputStream templateInputStream; /** - * Template file. - * This file is read into memory, excessive cases can lead to OOM. + * Template file, read through a bounded buffer when the writer is built. + * The parsed workbook still resides in memory, so large templates can lead to OOM. *
* If 'inputStream' and 'file' all not empty, file first */ private File templateFile; /** - * Default true. + * Whether to close caller-provided template input and output streams. Default true. + * Streams opened internally for files are always closed. */ private Boolean autoCloseStream; /** diff --git a/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/holder/WriteWorkbookHolder.java b/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/holder/WriteWorkbookHolder.java index 021dd8523c..6afcb444b9 100644 --- a/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/holder/WriteWorkbookHolder.java +++ b/fesod-sheet/src/main/java/org/apache/fesod/sheet/write/metadata/holder/WriteWorkbookHolder.java @@ -25,8 +25,9 @@ package org.apache.fesod.sheet.write.metadata.holder; -import java.io.ByteArrayInputStream; +import java.io.BufferedInputStream; import java.io.File; +import java.io.FileInputStream; import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; @@ -40,7 +41,7 @@ import lombok.Setter; import lombok.ToString.Exclude; import lombok.extern.slf4j.Slf4j; -import org.apache.fesod.common.util.IoUtils; +import org.apache.commons.io.input.CloseShieldInputStream; import org.apache.fesod.common.util.MapUtils; import org.apache.fesod.sheet.enums.CellDataTypeEnum; import org.apache.fesod.sheet.enums.HolderEnum; @@ -48,7 +49,6 @@ import org.apache.fesod.sheet.metadata.data.DataFormatData; import org.apache.fesod.sheet.support.ExcelTypeEnum; import org.apache.fesod.sheet.util.DateUtils; -import org.apache.fesod.sheet.util.FileUtils; import org.apache.fesod.sheet.util.StyleUtil; import org.apache.fesod.sheet.write.handler.context.WorkbookWriteHandlerContext; import org.apache.fesod.sheet.write.metadata.WriteWorkbook; @@ -129,9 +129,9 @@ public class WriteWorkbookHolder extends AbstractWriteHolder { */ private File templateFile; /** - * Temporary template file stream. + * Buffered template stream, closed after workbook creation. *
- * A temporary file stream needs to be created in order not to modify the original template file.
+ * Kept non-null after closing because it also marks the presence of a template.
*/
private InputStream tempTemplateInputStream;
/**
@@ -242,7 +242,7 @@ public WriteWorkbookHolder(WriteWorkbook writeWorkbook) {
// init handler
try {
initHandler(writeWorkbook, null);
- copyTemplate();
+ prepareTemplate();
} catch (IOException e) {
closeOutputStreamOnFailure();
throw new ExcelGenerateException("Copy template failure.", e);
@@ -274,13 +274,10 @@ public WriteWorkbookHolder(WriteWorkbook writeWorkbook) {
}
/**
- * Close the output stream when initialization fails, so that the target file is not held open.
- * Mirrors the close behavior of the success path, which is also gated by {@code autoCloseStream};
- * callers that opt out of automatic closing via {@code autoCloseStream(false)} keep that behavior
- * here as well.
+ * Close owned output streams when initialization fails, so that the target file is not held open.
*/
private void closeOutputStreamOnFailure() {
- if (autoCloseStream && outputStream != null) {
+ if (shouldCloseOutputStream() && outputStream != null) {
try {
outputStream.close();
} catch (IOException e) {
@@ -289,26 +286,28 @@ private void closeOutputStreamOnFailure() {
}
}
- private void copyTemplate() throws IOException {
+ /**
+ * Streams opened internally for files are always closed; caller-provided streams follow {@code autoCloseStream}.
+ */
+ public boolean shouldCloseOutputStream() {
+ return file != null || autoCloseStream;
+ }
+
+ private void prepareTemplate() throws IOException {
if (writeWorkbook.getTemplateFile() == null && writeWorkbook.getTemplateInputStream() == null) {
return;
}
if (this.excelType == ExcelTypeEnum.CSV) {
throw new ExcelGenerateException("csv cannot use template.");
}
- byte[] templateFileByte = null;
if (writeWorkbook.getTemplateFile() != null) {
- templateFileByte = FileUtils.readFileToByteArray(writeWorkbook.getTemplateFile());
- } else if (writeWorkbook.getTemplateInputStream() != null) {
- try {
- templateFileByte = IoUtils.toByteArray(writeWorkbook.getTemplateInputStream());
- } finally {
- if (autoCloseStream) {
- writeWorkbook.getTemplateInputStream().close();
- }
- }
+ this.tempTemplateInputStream =
+ new BufferedInputStream(new FileInputStream(writeWorkbook.getTemplateFile()));
+ } else {
+ InputStream template = writeWorkbook.getTemplateInputStream();
+ this.tempTemplateInputStream =
+ new BufferedInputStream(autoCloseStream ? template : CloseShieldInputStream.wrap(template));
}
- this.tempTemplateInputStream = new ByteArrayInputStream(templateFileByte);
}
@Override
diff --git a/fesod-sheet/src/test/java/org/apache/fesod/sheet/write/TemplateStreamBufferingTest.java b/fesod-sheet/src/test/java/org/apache/fesod/sheet/write/TemplateStreamBufferingTest.java
new file mode 100644
index 0000000000..4d49c197d5
--- /dev/null
+++ b/fesod-sheet/src/test/java/org/apache/fesod/sheet/write/TemplateStreamBufferingTest.java
@@ -0,0 +1,384 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.fesod.sheet.write;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.file.Files;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.concurrent.atomic.AtomicReference;
+import org.apache.fesod.sheet.ExcelWriter;
+import org.apache.fesod.sheet.FesodSheet;
+import org.apache.fesod.sheet.testkit.Tags;
+import org.apache.fesod.sheet.testkit.base.AbstractExcelTest;
+import org.apache.fesod.sheet.testkit.enums.ExcelFormat;
+import org.apache.fesod.sheet.testkit.params.ExcelFormatSource;
+import org.apache.fesod.sheet.testkit.params.FormatScope;
+import org.apache.fesod.sheet.write.builder.ExcelWriterBuilder;
+import org.apache.fesod.sheet.write.handler.WorkbookWriteHandler;
+import org.apache.fesod.sheet.write.handler.context.WorkbookWriteHandlerContext;
+import org.apache.fesod.sheet.write.metadata.WriteSheet;
+import org.apache.fesod.sheet.write.metadata.WriteWorkbook;
+import org.apache.fesod.sheet.write.metadata.fill.FillConfig;
+import org.apache.fesod.sheet.write.metadata.holder.WriteWorkbookHolder;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.ss.usermodel.WorkbookFactory;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+@Tag(Tags.WRITE)
+class TemplateStreamBufferingTest extends AbstractExcelTest {
+
+ @ParameterizedTest
+ @ValueSource(booleans = {true, false})
+ void preparingLargeTemplateDoesNotReadTheEntireInput(boolean autoClose) throws Exception {
+ GeneratedInputStream source = new GeneratedInputStream(32 * 1024 * 1024);
+ WriteWorkbook configuration = new WriteWorkbook();
+ configuration.setOutputStream(new ByteArrayOutputStream());
+ configuration.setTemplateInputStream(source);
+ configuration.setAutoCloseStream(autoClose);
+ WriteWorkbookHolder holder = new WriteWorkbookHolder(configuration);
+ try (InputStream prepared = holder.getTempTemplateInputStream()) {
+ Assertions.assertEquals(0, source.bytesRead, "preparing a template must not materialize its bytes");
+ Assertions.assertEquals(0, prepared.read());
+ Assertions.assertTrue(source.bytesRead < 32 * 1024 * 1024, "the first read must use bounded buffering");
+ }
+ }
+
+ @ParameterizedTest
+ @ExcelFormatSource(FormatScope.BINARY)
+ void fileAndStreamTemplatesRetainFillBehaviorAndOwnership(ExcelFormat format) throws Exception {
+ File template = template(format);
+ byte[] original = Files.readAllBytes(template.toPath());
+ for (boolean inMemory : new boolean[] {false, true}) {
+ File fileOutput = createTempFile("file-template", format);
+ fill(FesodSheet.write(fileOutput).withTemplate(template).inMemory(inMemory));
+ assertFilled(fileOutput);
+
+ for (boolean autoClose : new boolean[] {true, false}) {
+ File streamOutput = createTempFile("stream-template", format);
+ try (TrackingInputStream source = new TrackingInputStream(Files.newInputStream(template.toPath()))) {
+ ExcelWriterBuilder builder = FesodSheet.write(streamOutput)
+ .withTemplate(source)
+ .autoCloseStream(autoClose)
+ .inMemory(inMemory);
+ FileOutputStream output;
+ try (ExcelWriter writer = builder.build()) {
+ output = (FileOutputStream)
+ writer.writeContext().writeWorkbookHolder().getOutputStream();
+ Assertions.assertEquals(autoClose ? 1 : 0, source.closeCalls);
+ InputStream prepared =
+ writer.writeContext().writeWorkbookHolder().getTempTemplateInputStream();
+ Assertions.assertNotNull(prepared, "the closed stream still marks a template workbook");
+ Assertions.assertThrows(IOException.class, prepared::read);
+ fill(writer);
+ }
+ Assertions.assertFalse(output.getFD().valid(), "Fesod must close its own output stream");
+ Assertions.assertEquals(autoClose ? 1 : 0, source.closeCalls);
+ if (!autoClose) {
+ Assertions.assertEquals(-1, source.read());
+ }
+ }
+ assertFilled(streamOutput);
+ Files.delete(Files.move(
+ streamOutput.toPath(), streamOutput.toPath().resolveSibling("moved" + streamOutput.getName())));
+ }
+ }
+ Assertions.assertArrayEquals(original, Files.readAllBytes(template.toPath()));
+ }
+
+ @ParameterizedTest
+ @ExcelFormatSource(FormatScope.BINARY)
+ void finishingHonorsCallerOutputStreamOwnership(ExcelFormat format) throws Exception {
+ File template = template(format);
+ for (boolean inMemory : new boolean[] {false, true}) {
+ for (boolean autoClose : new boolean[] {true, false}) {
+ TrackingOutputStream output = new TrackingOutputStream();
+ try (TrackingInputStream source = new TrackingInputStream(Files.newInputStream(template.toPath()))) {
+ fill(FesodSheet.write(output)
+ .excelType(format.toExcelTypeEnum())
+ .withTemplate(source)
+ .autoCloseStream(autoClose)
+ .inMemory(inMemory));
+ Assertions.assertEquals(autoClose ? 1 : 0, source.closeCalls);
+ Assertions.assertEquals(autoClose ? 1 : 0, output.closeCalls);
+ }
+ }
+ }
+ }
+
+ @ParameterizedTest
+ @ExcelFormatSource(FormatScope.BINARY)
+ void parsingFailureClosesFileOutputRegardlessOfAutoClose(ExcelFormat format) throws Exception {
+ for (boolean autoClose : new boolean[] {true, false}) {
+ File output = createTempFile("invalid-template", format);
+ AtomicReference