From 19f7c83812a4c81227601ff1abadca84beb05cb9 Mon Sep 17 00:00:00 2001 From: Houston Putman Date: Tue, 3 Nov 2026 14:37:41 -0800 Subject: [PATCH 1/2] SOLR-18618: Upload configsets and configset files from a backup repository --- ...18618-configset-upload-from-repository.yml | 7 + .../client/api/endpoint/ConfigsetsApi.java | 52 +++- .../solr/handler/admin/ConfigSetsHandler.java | 25 +- .../handler/configsets/UploadConfigSet.java | 225 +++++++++++++--- ...alFSConfigSetUploadFromRepositoryTest.java | 96 +++++++ .../configsets/UploadConfigSetAPITest.java | 48 +++- .../UploadConfigSetFileAPITest.java | 33 ++- .../GCSConfigSetUploadFromRepositoryTest.java | 62 +++++ .../S3ConfigSetUploadFromRepositoryTest.java | 79 ++++++ .../pages/configsets-api.adoc | 38 +++ ...ractConfigSetUploadFromRepositoryTest.java | 243 ++++++++++++++++++ 11 files changed, 851 insertions(+), 57 deletions(-) create mode 100644 changelog/unreleased/SOLR-18618-configset-upload-from-repository.yml create mode 100644 solr/core/src/test/org/apache/solr/cloud/LocalFSConfigSetUploadFromRepositoryTest.java create mode 100644 solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSConfigSetUploadFromRepositoryTest.java create mode 100644 solr/modules/s3-repository/src/test/org/apache/solr/s3/S3ConfigSetUploadFromRepositoryTest.java create mode 100644 solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractConfigSetUploadFromRepositoryTest.java diff --git a/changelog/unreleased/SOLR-18618-configset-upload-from-repository.yml b/changelog/unreleased/SOLR-18618-configset-upload-from-repository.yml new file mode 100644 index 000000000000..fba07df940dd --- /dev/null +++ b/changelog/unreleased/SOLR-18618-configset-upload-from-repository.yml @@ -0,0 +1,7 @@ +title: ConfigSet upload APIs can read a configset, or a single file, from a backup repository such as S3 or GCS. +type: added +authors: + - name: Houston Putman +links: + - name: SOLR-18618 + url: https://issues.apache.org/jira/browse/SOLR-18618 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java index f7fd006ef440..0caa3eaede19 100644 --- a/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/ConfigsetsApi.java @@ -20,6 +20,7 @@ import static org.apache.solr.client.api.util.Constants.RAW_OUTPUT_PROPERTY; import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; import io.swagger.v3.oas.annotations.extensions.Extension; import io.swagger.v3.oas.annotations.extensions.ExtensionProperty; import io.swagger.v3.oas.annotations.parameters.RequestBody; @@ -119,18 +120,41 @@ StreamingOutput getConfigSetFile( } /** - * V2 API definition for uploading an entire configset as a ZIP archive. + * V2 API definition for uploading an entire configset. + * + *

The configset is read either from the ZIP request body, or from a directory in a backup + * repository such as S3 or GCS, identified by {@code repository} and {@code location}. The two + * sources are mutually exclusive. * *

Equivalent to the existing v1 API /admin/configs?action=UPLOAD */ @Path("/configsets/{configSetName}") interface Upload { @PUT - @Operation(summary = "Upload a configset as a ZIP archive.", tags = "configsets") + @Operation( + summary = "Upload a configset as a ZIP archive or from a backup repository.", + description = + "Provide exactly one source: either a ZIP archive as the request body, or" + + " 'repository' and 'location' (both required if either is given) naming a" + + " directory in a backup repository, with an empty request body. Providing both is" + + " an error.", + tags = "configsets") SolrJerseyResponse uploadConfigSet( @PathParam("configSetName") String configSetName, @QueryParam("overwrite") Boolean overwrite, @QueryParam("cleanup") Boolean cleanup, + @Parameter( + description = + "Backup repository to read the configset from. Required if 'location' is" + + " given; mutually exclusive with a request body.") + @QueryParam("repository") + String repository, + @Parameter( + description = + "Directory in the backup repository holding the configset files. Required if" + + " 'repository' is given; mutually exclusive with a request body.") + @QueryParam("location") + String location, @RequestBody( required = true, extensions = { @@ -147,16 +171,36 @@ SolrJerseyResponse uploadConfigSet( * V2 API definition for putting a single file to an existing configset. * *

This endpoint allows updating individual configuration files without re-uploading the entire - * configset. The file path is specified as part of the URL path. + * configset. The file path is specified as part of the URL path. The file contents are either the + * request body, or a file in a backup repository identified by {@code repository} and {@code + * location}. */ @Path("/configsets/{configSetName}") interface PutFile { @PUT @Path("/files/{filePath:.+}") - @Operation(summary = "Upload a single file to a configset.", tags = "configsets") + @Operation( + summary = "Upload a single file to a configset.", + description = + "Provide exactly one source: either the file contents as the request body, or" + + " 'repository' and 'location' (both required if either is given) naming a file in" + + " a backup repository, with an empty request body. Providing both is an error.", + tags = "configsets") SolrJerseyResponse uploadConfigSetFile( @PathParam("configSetName") String configSetName, @PathParam("filePath") String filePath, + @Parameter( + description = + "Backup repository to read the file from. Required if 'location' is given;" + + " mutually exclusive with a request body.") + @QueryParam("repository") + String repository, + @Parameter( + description = + "Path of the file in the backup repository. Required if 'repository' is given;" + + " mutually exclusive with a request body.") + @QueryParam("location") + String location, @RequestBody( required = true, extensions = { diff --git a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java index 50159bf1abd3..2c8d05a177ab 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/ConfigSetsHandler.java @@ -31,6 +31,7 @@ import org.apache.solr.common.SolrException.ErrorCode; import org.apache.solr.common.params.ConfigSetParams; import org.apache.solr.common.params.ConfigSetParams.ConfigSetAction; +import org.apache.solr.common.params.CoreAdminParams; import org.apache.solr.common.params.SolrParams; import org.apache.solr.core.CoreContainer; import org.apache.solr.handler.RequestHandlerBase; @@ -96,13 +97,22 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw final var configSetName = req.getParams().required().get(NAME); final var overwrite = req.getParams().getBool(ConfigSetParams.OVERWRITE, false); final var cleanup = req.getParams().getBool(ConfigSetParams.CLEANUP, false); - final var configSetData = ConfigSetAPIBase.ensureNonEmptyInputStream(req); SolrJerseyResponse uploadResponse; - if (req.getParams() - .get(ConfigSetParams.FILE_PATH, "") - .isEmpty()) { // Uploading a whole configset + final var repository = req.getParams().get(CoreAdminParams.BACKUP_REPOSITORY); + final var location = req.getParams().get(CoreAdminParams.BACKUP_LOCATION); + if ((repository == null) != (location == null)) { + throw new SolrException( + ErrorCode.BAD_REQUEST, + "The 'repository' and 'location' parameters must be provided together."); + } + final var fromRepository = repository != null; + final var filePath = req.getParams().get(ConfigSetParams.FILE_PATH, ""); + final var configSetData = + fromRepository ? null : ConfigSetAPIBase.ensureNonEmptyInputStream(req); + if (filePath.isEmpty()) { // Uploading a whole configset uploadResponse = - uploadApi.uploadConfigSet(configSetName, overwrite, cleanup, configSetData); + uploadApi.uploadConfigSet( + configSetName, overwrite, cleanup, repository, location, configSetData); } else { // Uploading a single file // Single file uploads do not support cleanup or overwrite parameters if (cleanup) { @@ -111,8 +121,9 @@ public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throw "ConfigSet uploads do not allow cleanup=true when filePath is used."); } // Note: overwrite parameter is ignored for single file uploads (always overwrites) - final var filePath = req.getParams().get(ConfigSetParams.FILE_PATH); - uploadResponse = uploadApi.uploadConfigSetFile(configSetName, filePath, configSetData); + uploadResponse = + uploadApi.uploadConfigSetFile( + configSetName, filePath, repository, location, configSetData); } V2ApiUtils.squashIntoSolrResponseWithoutHeader(rsp, uploadResponse); break; diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java index bb9ca94c761a..1a210348ebfb 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/UploadConfigSet.java @@ -22,21 +22,27 @@ import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; +import java.net.URI; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.StandardCopyOption; import java.util.ArrayList; import java.util.Enumeration; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.zip.ZipEntry; import java.util.zip.ZipException; import java.util.zip.ZipFile; +import org.apache.lucene.store.IOContext; +import org.apache.lucene.store.IndexInput; import org.apache.solr.client.api.endpoint.ConfigsetsApi; import org.apache.solr.client.api.model.SolrJerseyResponse; import org.apache.solr.client.solrj.util.SolrIdentifierValidator; import org.apache.solr.common.SolrException; import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.CoreContainer; +import org.apache.solr.core.backup.repository.BackupRepository; import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; @@ -65,30 +71,47 @@ public UploadConfigSet( @Override @PermissionName(CONFIG_EDIT_PERM) public SolrJerseyResponse uploadConfigSet( - String configSetName, Boolean overwrite, Boolean cleanup, InputStream requestBody) + String configSetName, + Boolean overwrite, + Boolean cleanup, + String repository, + String location, + InputStream requestBody) throws IOException { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); ensureConfigSetUploadEnabled(); + final boolean fromRepository = isFromRepository(repository, location); + ensureSingleSource(fromRepository, requestBody); + if (!fromRepository && requestBody == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "A configset upload must provide either a ZIP request body or a repository and location."); + } SolrIdentifierValidator.validateConfigSetName(configSetName); - boolean overwritesExisting = configSetService.checkConfigExists(configSetName); - // Get upload parameters - if (overwrite == null) overwrite = true; - if (cleanup == null) cleanup = false; - - if (overwritesExisting && !overwrite) { + final boolean overwritesExisting = configSetService.checkConfigExists(configSetName); + if (overwritesExisting && overwrite != null && !overwrite) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "The configuration " + configSetName + " already exists"); } + final List filesToDelete = + overwritesExisting && cleanup != null && cleanup + ? configSetService.getAllConfigFiles(configSetName) + : new ArrayList<>(); - List filesToDelete; - if (overwritesExisting && cleanup) { - filesToDelete = configSetService.getAllConfigFiles(configSetName); + if (fromRepository) { + uploadFromRepository(configSetName, repository, location, filesToDelete); } else { - filesToDelete = new ArrayList<>(); + uploadZip(configSetName, requestBody, filesToDelete); } + deleteUnusedFiles(configSetService, configSetName, filesToDelete); + return response; + } + + private void uploadZip(String configSetName, InputStream requestBody, List filesToDelete) + throws IOException { // Write the request body to a temp file so we can use ZipFile, which reads the central // directory and correctly handles entries that use the STORED method with an EXT (data // descriptor) flag — a combination that ZipInputStream cannot process. This allows @@ -125,48 +148,184 @@ public SolrJerseyResponse uploadConfigSet( } finally { Files.deleteIfExists(tempZip); } - deleteUnusedFiles(configSetService, configSetName, filesToDelete); + } - return response; + private void uploadFromRepository( + String configSetName, + String repositoryName, + String requestedLocation, + List filesToDelete) + throws IOException { + try (BackupRepository repository = coreContainer.newBackupRepository(repositoryName)) { + final String location = resolveRepositoryLocation(repository, requestedLocation); + final URI source = repository.createDirectoryURI(location); + if (!repository.exists(source)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "The location " + source + " does not exist in the backup repository"); + } + + // Read everything before writing so a failure doesn't leave a half-uploaded configset. + final Map files = new LinkedHashMap<>(); + readConfigFilesFromRepository(repository, source, "", files); + if (files.isEmpty()) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "No configset files were found at " + source + " in the backup repository"); + } + + for (var file : files.entrySet()) { + filesToDelete.remove(file.getKey()); + configSetService.uploadFileToConfig(configSetName, file.getKey(), file.getValue(), true); + } + } + } + + /** Whether the upload reads from a backup repository; 'repository' and 'location' go together. */ + private static boolean isFromRepository(String repository, String location) { + if ((repository == null) != (location == null)) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "The 'repository' and 'location' parameters must be provided together."); + } + return repository != null; + } + + private static void ensureSingleSource(boolean fromRepository, InputStream requestBody) + throws IOException { + if (fromRepository && requestBody != null && requestBody.available() > 0) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "An upload must provide either a request body or a repository location, not both."); + } + } + + private static void readConfigFilesFromRepository( + BackupRepository repository, URI dir, String prefix, Map files) + throws IOException { + for (String name : repository.listAll(dir)) { + final String filePath = prefix + name; + switch (repository.getPathType(repository.resolve(dir, name))) { + case DIRECTORY: + if (!name.startsWith(".")) { + readConfigFilesFromRepository( + repository, repository.resolveDirectory(dir, name), filePath + "/", files); + } + break; + case FILE: + if (ConfigSetService.isFileForbiddenInConfigSets(filePath)) { + log.warn("Not uploading file from repository, as it is a forbidden type: {}", filePath); + break; + } + final byte[] data = readRepositoryFile(repository, dir, name); + if (FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { + log.warn( + "Not uploading file {} from repository, as it matched the MAGIC signature of a forbidden mime type", + filePath); + } else { + files.put(filePath, data); + } + break; + } + } + } + + private String resolveRepositoryLocation(BackupRepository repository, String requestedLocation) { + final String location = repository.getBackupLocation(requestedLocation); + if (location == null) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "'location' is not specified as a parameter or as a default repository property"); + } + if ("file".equals(repository.createURI("x").getScheme())) { + coreContainer.assertPathAllowed(Path.of(location)); + } + return location; + } + + private static byte[] readRepositoryFile(BackupRepository repository, URI dir, String name) + throws IOException { + try (IndexInput in = repository.openInput(dir, name, IOContext.DEFAULT)) { + final byte[] data = new byte[(int) in.length()]; + in.readBytes(data, 0, data.length); + return data; + } } @Override @PermissionName(CONFIG_EDIT_PERM) public SolrJerseyResponse uploadConfigSetFile( - String configSetName, String filePath, InputStream requestBody) throws IOException { + String configSetName, + String filePath, + String repository, + String location, + InputStream requestBody) + throws IOException { final var response = instantiateJerseyResponse(SolrJerseyResponse.class); ensureConfigSetUploadEnabled(); + final boolean fromRepository = isFromRepository(repository, location); + ensureSingleSource(fromRepository, requestBody); SolrIdentifierValidator.validateConfigSetName(configSetName); + ensureValidFilePath(filePath); + storeFile( + configSetName, + filePath, + fromRepository + ? readSingleFileFromRepository(repository, location) + : requestBody.readAllBytes()); + return response; + } - boolean overwrite = true; - - // Get upload parameters - - String singleFilePath = filePath != null ? filePath : ""; - - String fixedSingleFilePath = singleFilePath; - if (!fixedSingleFilePath.isEmpty() && fixedSingleFilePath.charAt(0) == '/') { - fixedSingleFilePath = fixedSingleFilePath.substring(1); - } - byte[] data = requestBody.readAllBytes(); + private static String ensureValidFilePath(String filePath) { + final String singleFilePath = filePath != null ? filePath : ""; + final String fixedSingleFilePath = + !singleFilePath.isEmpty() && singleFilePath.charAt(0) == '/' + ? singleFilePath.substring(1) + : singleFilePath; if (fixedSingleFilePath.isEmpty()) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "The file path provided for upload, '" + singleFilePath + "', is not valid."); - } else if (ConfigSetService.isFileForbiddenInConfigSets(fixedSingleFilePath) + } + return fixedSingleFilePath; + } + + private void storeFile(String configSetName, String filePath, byte[] data) throws IOException { + final String fixedSingleFilePath = ensureValidFilePath(filePath); + if (ConfigSetService.isFileForbiddenInConfigSets(fixedSingleFilePath) || FileTypeMagicUtil.isFileForbiddenInConfigset(data)) { throw new SolrException( SolrException.ErrorCode.BAD_REQUEST, "The file type provided for upload, '" - + singleFilePath + + filePath + "', is forbidden for use in configSets."); - } else { - // Create a node for the configuration in config - // For creating the baseNode, the cleanup parameter is only allowed to be true when - // singleFilePath is not passed. - configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, overwrite); } - return response; + configSetService.uploadFileToConfig(configSetName, fixedSingleFilePath, data, true); + } + + private byte[] readSingleFileFromRepository(String repositoryName, String requestedLocation) + throws IOException { + try (BackupRepository repository = coreContainer.newBackupRepository(repositoryName)) { + final String location = resolveRepositoryLocation(repository, requestedLocation); + final String trimmed = + location.endsWith("/") ? location.substring(0, location.length() - 1) : location; + final int slash = trimmed.lastIndexOf('/'); + if (slash < 0 || slash == trimmed.length() - 1) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "The location '" + location + "' must be the path of a file in the backup repository"); + } + final URI dir = repository.createDirectoryURI(slash == 0 ? "/" : trimmed.substring(0, slash)); + final String name = trimmed.substring(slash + 1); + final URI file = repository.resolve(dir, name); + if (!repository.exists(file) + || repository.getPathType(file) != BackupRepository.PathType.FILE) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "The location " + file + " is not a file in the backup repository"); + } + return readRepositoryFile(repository, dir, name); + } } private void deleteUnusedFiles( diff --git a/solr/core/src/test/org/apache/solr/cloud/LocalFSConfigSetUploadFromRepositoryTest.java b/solr/core/src/test/org/apache/solr/cloud/LocalFSConfigSetUploadFromRepositoryTest.java new file mode 100644 index 000000000000..6c2f92b0caa0 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/cloud/LocalFSConfigSetUploadFromRepositoryTest.java @@ -0,0 +1,96 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud; + +import java.net.URI; +import java.nio.file.Path; +import org.apache.solr.cloud.api.collections.AbstractConfigSetUploadFromRepositoryTest; +import org.apache.solr.cloud.api.collections.AbstractInstallShardTest; +import org.eclipse.jetty.client.ContentResponse; +import org.eclipse.jetty.client.StringRequestContent; +import org.eclipse.jetty.http.HttpMethod; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Tests uploading configsets from a backup repository with {@link + * org.apache.solr.core.backup.repository.LocalFileSystemRepository}. + */ +public class LocalFSConfigSetUploadFromRepositoryTest + extends AbstractConfigSetUploadFromRepositoryTest { + + private static final String REPOSITORY_NAME = "local"; + + private static final String SOLR_XML = + AbstractInstallShardTest.defaultSolrXmlTextWithBackupRepository( + "${solr.security.allow.paths:}\n" + + "\n" + + " \n" + + "\n"); + + private static Path repositoryDir; + + @BeforeClass + public static void setupClass() throws Exception { + System.setProperty("solr.security.allow.paths", "*"); + repositoryDir = createTempDir("repository"); + configureCluster(1).withSolrXml(SOLR_XML).configure(); + } + + @Override + protected String repositoryName() { + return REPOSITORY_NAME; + } + + @Override + protected String baseLocation() { + return repositoryDir.toString(); + } + + @Override + protected String locationParam(URI uri) { + return Path.of(uri).toString(); + } + + @Test + public void testRawBodiesOfAnyContentTypeAreStillStored() throws Exception { + // A JSON config file is stored as-is + final var jsonFile = put("rawjson/files/params.json", "application/json", "{\"a\":1}"); + assertEquals(jsonFile.getContentAsString(), 200, jsonFile.getStatus()); + assertTrue(cluster.getZkClient().exists("/configs/rawjson/params.json")); + + // Clients such as curl --data-binary send this content type by default + final var form = + put("rawform/files/solrconfig.xml", "application/x-www-form-urlencoded", ""); + assertEquals(form.getContentAsString(), 200, form.getStatus()); + assertTrue(cluster.getZkClient().exists("/configs/rawform/solrconfig.xml")); + } + + private ContentResponse put(String path, String contentType, String body) throws Exception { + final var jetty = cluster.getJettySolrRunners().getFirst(); + return jetty + .getSolrClient() + .getHttpClient() + .newRequest( + jetty.getBaseUrl().toString().replaceAll("/solr$", "") + "/api/configsets/" + path) + .method(HttpMethod.PUT) + .body(new StringRequestContent(contentType, body)) + .send(); + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java b/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java index b376a2592b19..867ca4452efc 100644 --- a/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetAPITest.java @@ -18,6 +18,8 @@ package org.apache.solr.handler.configsets; import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -31,8 +33,10 @@ import java.util.zip.ZipOutputStream; import org.apache.solr.SolrTestCase; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.FileSystemConfigSetService; +import org.apache.solr.core.backup.repository.LocalFileSystemRepository; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -104,7 +108,7 @@ public void testSuccessfulZipUpload() throws Exception { InputStream zipStream = createZipStream("solrconfig.xml", ""); final var api = new UploadConfigSet(mockCoreContainer, null, null); - final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + final var response = api.uploadConfigSet(configSetName, true, false, null, null, zipStream); assertNotNull(response); assertTrue( @@ -125,7 +129,7 @@ public void testSuccessfulZipUploadWithMultipleFiles() throws Exception { "stopwords.txt", "a\nthe"); final var api = new UploadConfigSet(mockCoreContainer, null, null); - final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + final var response = api.uploadConfigSet(configSetName, true, false, null, null, zipStream); assertNotNull(response); assertTrue(configSetService.checkConfigExists(configSetName)); @@ -148,7 +152,8 @@ public void testEmptyZipThrowsBadRequest() throws Exception { final var api = new UploadConfigSet(mockCoreContainer, null, null); final var ex = assertThrows( - SolrException.class, () -> api.uploadConfigSet("newconfig", true, false, emptyZip)); + SolrException.class, + () -> api.uploadConfigSet("newconfig", true, false, null, null, emptyZip)); assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); assertTrue( @@ -165,7 +170,8 @@ public void testNonZipDataThrowsBadRequest() { final var api = new UploadConfigSet(mockCoreContainer, null, null); // This should fail either as bad ZIP or as empty ZIP - assertThrows(Exception.class, () -> api.uploadConfigSet("newconfig", true, false, notAZip)); + assertThrows( + Exception.class, () -> api.uploadConfigSet("newconfig", true, false, null, null, notAZip)); } @Test @@ -177,7 +183,7 @@ public void testOverwriteExistingConfigSet() throws Exception { // Upload new content with overwrite=true InputStream zipStream = createZipStream("solrconfig.xml", ""); final var api = new UploadConfigSet(mockCoreContainer, null, null); - final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + final var response = api.uploadConfigSet(configSetName, true, false, null, null, zipStream); assertNotNull(response); @@ -197,7 +203,7 @@ public void testOverwriteFalseThrowsExceptionWhenExists() throws Exception { final var ex = assertThrows( SolrException.class, - () -> api.uploadConfigSet(configSetName, false, false, zipStream)); + () -> api.uploadConfigSet(configSetName, false, false, null, null, zipStream)); assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); assertTrue( @@ -222,7 +228,7 @@ public void testCleanupRemovesUnusedFiles() throws Exception { // Upload new ZIP with only one file and cleanup=true InputStream zipStream = createZipStream("solrconfig.xml", ""); final var api = new UploadConfigSet(mockCoreContainer, null, null); - final var response = api.uploadConfigSet(configSetName, true, true, zipStream); + final var response = api.uploadConfigSet(configSetName, true, true, null, null, zipStream); assertNotNull(response); @@ -251,7 +257,7 @@ public void testCleanupFalseKeepsExistingFiles() throws Exception { // Upload new ZIP with only one file and cleanup=false InputStream zipStream = createZipStream("solrconfig.xml", ""); final var api = new UploadConfigSet(mockCoreContainer, null, null); - final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + final var response = api.uploadConfigSet(configSetName, true, false, null, null, zipStream); assertNotNull(response); @@ -271,7 +277,7 @@ public void testDefaultParametersWhenNull() throws Exception { final var api = new UploadConfigSet(mockCoreContainer, null, null); // Pass null for overwrite and cleanup - should use defaults (overwrite=true, cleanup=false) - final var response = api.uploadConfigSet(configSetName, null, null, zipStream); + final var response = api.uploadConfigSet(configSetName, null, null, null, null, zipStream); assertNotNull(response); assertTrue(configSetService.checkConfigExists(configSetName)); @@ -294,7 +300,7 @@ public void testZipWithDirectoryEntries() throws Exception { InputStream zipStream = new ByteArrayInputStream(baos.toByteArray()); final var api = new UploadConfigSet(mockCoreContainer, null, null); - final var response = api.uploadConfigSet(configSetName, true, false, zipStream); + final var response = api.uploadConfigSet(configSetName, true, false, null, null, zipStream); assertNotNull(response); assertTrue(configSetService.checkConfigExists(configSetName)); @@ -304,4 +310,26 @@ public void testZipWithDirectoryEntries() throws Exception { configSetService.downloadFileFromConfig(configSetName, "conf/solrconfig.xml"); assertEquals("", new String(uploadedData, StandardCharsets.UTF_8)); } + + @Test + public void testRepositoryLocationMustBeAllowedPath() throws Exception { + final Path disallowed = createTempDir("disallowed"); + final var repository = new LocalFileSystemRepository(); + repository.init(new NamedList<>()); + when(mockCoreContainer.newBackupRepository(any())).thenReturn(repository); + doThrow(new SolrException(SolrException.ErrorCode.FORBIDDEN, "path not allowed")) + .when(mockCoreContainer) + .assertPathAllowed(any()); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var e = + expectThrows( + SolrException.class, + () -> + api.uploadConfigSet( + "newconfig", true, false, "local", disallowed.toString(), null)); + + assertEquals(SolrException.ErrorCode.FORBIDDEN.code, e.code()); + assertFalse(configSetService.checkConfigExists("newconfig")); + } } diff --git a/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetFileAPITest.java b/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetFileAPITest.java index c58305b5fd35..fdec55660402 100644 --- a/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetFileAPITest.java +++ b/solr/core/src/test/org/apache/solr/handler/configsets/UploadConfigSetFileAPITest.java @@ -18,6 +18,8 @@ package org.apache.solr.handler.configsets; import static org.apache.solr.SolrTestCaseJ4.assumeWorkingMockito; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doThrow; import static org.mockito.Mockito.mock; import static org.mockito.Mockito.when; @@ -27,8 +29,10 @@ import java.nio.file.Path; import org.apache.solr.SolrTestCase; import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.FileSystemConfigSetService; +import org.apache.solr.core.backup.repository.LocalFileSystemRepository; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -62,7 +66,7 @@ public void testSingleFileUploadSuccess() throws Exception { InputStream fileStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); final var api = new UploadConfigSet(mockCoreContainer, null, null); - api.uploadConfigSetFile(configSetName, filePath, fileStream); + api.uploadConfigSetFile(configSetName, filePath, null, null, fileStream); // Verify the file was uploaded byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, filePath); @@ -79,7 +83,8 @@ public void testSingleFileWithEmptyPathThrowsBadRequest() { // Test with empty filePath final var ex = assertThrows( - SolrException.class, () -> api.uploadConfigSetFile(configSetName, "", fileStream)); + SolrException.class, + () -> api.uploadConfigSetFile(configSetName, "", null, null, fileStream)); assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); assertTrue("Error should mention invalid path", ex.getMessage().contains("not valid")); } @@ -92,10 +97,32 @@ public void testSingleFileUploadWithNestedPath() throws Exception { InputStream fileStream = new ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8)); final var api = new UploadConfigSet(mockCoreContainer, null, null); - api.uploadConfigSetFile(configSetName, filePath, fileStream); + api.uploadConfigSetFile(configSetName, filePath, null, null, fileStream); // Verify the file was uploaded with correct path byte[] uploadedData = configSetService.downloadFileFromConfig(configSetName, filePath); assertEquals(content, new String(uploadedData, StandardCharsets.UTF_8)); } + + @Test + public void testRepositoryLocationMustBeAllowedPath() throws Exception { + final Path disallowed = createTempDir("disallowed").resolve("solrconfig.xml"); + final var repository = new LocalFileSystemRepository(); + repository.init(new NamedList<>()); + when(mockCoreContainer.newBackupRepository(any())).thenReturn(repository); + doThrow(new SolrException(SolrException.ErrorCode.FORBIDDEN, "path not allowed")) + .when(mockCoreContainer) + .assertPathAllowed(any()); + + final var api = new UploadConfigSet(mockCoreContainer, null, null); + final var e = + expectThrows( + SolrException.class, + () -> + api.uploadConfigSetFile( + "newconfig", "solrconfig.xml", "local", disallowed.toString(), null)); + + assertEquals(SolrException.ErrorCode.FORBIDDEN.code, e.code()); + assertFalse(configSetService.checkConfigExists("newconfig")); + } } diff --git a/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSConfigSetUploadFromRepositoryTest.java b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSConfigSetUploadFromRepositoryTest.java new file mode 100644 index 000000000000..5981521fd471 --- /dev/null +++ b/solr/modules/gcs-repository/src/test/org/apache/solr/gcs/GCSConfigSetUploadFromRepositoryTest.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.gcs; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakLingering; +import org.apache.solr.cloud.api.collections.AbstractConfigSetUploadFromRepositoryTest; +import org.apache.solr.cloud.api.collections.AbstractInstallShardTest; +import org.junit.AfterClass; +import org.junit.BeforeClass; + +/** Tests uploading configsets from a backup repository with {@link GCSBackupRepository}. */ +@ThreadLeakLingering(linger = 10) +public class GCSConfigSetUploadFromRepositoryTest + extends AbstractConfigSetUploadFromRepositoryTest { + + private static final String REPOSITORY_NAME = "localfs"; + + private static final String BACKUP_REPOSITORY_XML = + " \n" + + " \n" + + " someBucketName\n" + + " backup1\n" + + " \n" + + " \n"; + + @BeforeClass + public static void setupClass() throws Exception { + configureCluster(1) + .withSolrXml( + AbstractInstallShardTest.defaultSolrXmlTextWithBackupRepository(BACKUP_REPOSITORY_XML)) + .configure(); + } + + @AfterClass + public static void tearDownClass() { + LocalStorageGCSBackupRepository.clearStashedStorage(); + } + + @Override + protected String repositoryName() { + return REPOSITORY_NAME; + } + + @Override + protected String baseLocation() { + return "backup1"; + } +} diff --git a/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3ConfigSetUploadFromRepositoryTest.java b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3ConfigSetUploadFromRepositoryTest.java new file mode 100644 index 000000000000..ab5649fa8894 --- /dev/null +++ b/solr/modules/s3-repository/src/test/org/apache/solr/s3/S3ConfigSetUploadFromRepositoryTest.java @@ -0,0 +1,79 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.s3; + +import com.carrotsearch.randomizedtesting.annotations.ThreadLeakFilters; +import org.apache.lucene.tests.util.QuickPatchThreadsFilter; +import org.apache.solr.SolrIgnoredThreadsFilter; +import org.apache.solr.cloud.api.collections.AbstractConfigSetUploadFromRepositoryTest; +import org.apache.solr.cloud.api.collections.AbstractInstallShardTest; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import software.amazon.awssdk.regions.Region; + +/** Tests uploading configsets from a backup repository with {@link S3BackupRepository}. */ +@ThreadLeakFilters( + filters = { + SolrIgnoredThreadsFilter.class, + QuickPatchThreadsFilter.class, + S3MockTestcontainersThreadFilter.class + }) +public class S3ConfigSetUploadFromRepositoryTest extends AbstractConfigSetUploadFromRepositoryTest { + + private static final String BUCKET_NAME = + S3ConfigSetUploadFromRepositoryTest.class.getSimpleName(); + + private static final String REPOSITORY_NAME = "s3"; + + private static final String BACKUP_REPOSITORY_XML = + " \n" + + " \n" + + " BUCKET\n" + + " REGION\n" + + " ENDPOINT\n" + + " \n" + + " \n"; + + @ClassRule + public static final S3MockContainerRule s3MockContainer = new S3MockContainerRule(BUCKET_NAME); + + @BeforeClass + public static void setupClass() throws Exception { + System.setProperty("aws.accessKeyId", "foo"); + System.setProperty("aws.secretAccessKey", "bar"); + + AbstractS3ClientTest.setS3ConfFile(); + + configureCluster(1) + .withSolrXml( + AbstractInstallShardTest.defaultSolrXmlTextWithBackupRepository(BACKUP_REPOSITORY_XML) + .replace("BUCKET", BUCKET_NAME) + .replace("REGION", Region.US_EAST_1.id()) + .replace("ENDPOINT", s3MockContainer.getHttpEndpoint())) + .configure(); + } + + @Override + protected String repositoryName() { + return REPOSITORY_NAME; + } + + @Override + protected String baseLocation() { + return "/"; + } +} diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc index f71186007d67..302b4ac747fc 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc @@ -167,6 +167,40 @@ When overwriting an existing configset (`overwrite=true`), this parameter tells When uploading an entire configset, the body of the request should be a zip file that contains the configset. The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file). +[[configsets-upload-repository]] +=== Upload from a Backup Repository + +Instead of a zip file, the configset can be read from a directory in a xref:deployment-guide:backup-restore.adoc#backuprestore-storage-repositories[backup repository] (such as S3 or GCS), with `solrconfig.xml` at its top level. +Pass these parameters and no request body: + +`repository`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The name of the backup repository. Required if `location` is given. + +`location`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +The directory in the repository that holds the configset's files. Required if `repository` is given. + +Providing both a request body and `repository`/`location` is an error. +The `overwrite` and `cleanup` parameters work the same way as for zip uploads. + +[source,bash] +---- +curl -X PUT "http://localhost:8983/api/configsets/myConfigSet?repository=s3&location=/configsets/myconfig" +---- + +For the v1 API, use `/solr/admin/configs?action=UPLOAD&name=myConfigSet&repository=s3&location=/configsets/myconfig`. + Here is an example on how to create the zip file named "myconfig.zip" and upload it as a configset named "myConfigSet": [tabs#uploadconfigset] @@ -236,6 +270,10 @@ s|Required |Default: none The path to the file within the configset, specified as part of the URL path (e.g., `solrconfig.xml` or `lang/stopwords_en.txt`). For nested paths, use slashes as shown in the examples below. +As when xref:#configsets-upload-repository[uploading a whole configset], the file can instead be read from a backup repository. +Pass the `repository` and `location` request parameters, where `location` is the path of a single file in the repository, and send no request body. +Both parameters are required if either is given. + Here is an example on how to upload a single file to a configset named "myConfigSet": [tabs#uploadsinglefile] diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractConfigSetUploadFromRepositoryTest.java b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractConfigSetUploadFromRepositoryTest.java new file mode 100644 index 000000000000..faae027b5a18 --- /dev/null +++ b/solr/test-framework/src/java/org/apache/solr/cloud/api/collections/AbstractConfigSetUploadFromRepositoryTest.java @@ -0,0 +1,243 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.cloud.api.collections; + +import static java.nio.charset.StandardCharsets.UTF_8; + +import java.io.ByteArrayInputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.net.URI; +import java.util.UUID; +import org.apache.solr.client.solrj.RemoteSolrException; +import org.apache.solr.client.solrj.request.ConfigsetsApi; +import org.apache.solr.cloud.SolrCloudTestCase; +import org.apache.solr.core.backup.repository.BackupRepository; +import org.junit.Test; + +/** + * Base class for testing configset uploads from a backup repository with various repository types. + * + *

Subclasses bootstrap a Solr cluster with a single configured backup repository, and tell this + * class its name and where in it test data can be written. This class stages the files through the + * cluster's own {@link BackupRepository}, so it works for any repository implementation. + * + * @see org.apache.solr.handler.configsets.UploadConfigSet + */ +public abstract class AbstractConfigSetUploadFromRepositoryTest extends SolrCloudTestCase { + + /** The name of the backup repository, as configured in solr.xml, to read from. */ + protected abstract String repositoryName(); + + /** A location in the repository under which each test stages its own files. */ + protected abstract String baseLocation(); + + /** The 'location' parameter that refers to {@code uri}; some repositories want a plain path. */ + protected String locationParam(URI uri) { + return uri.toString(); + } + + /** Writes a file, and any parent directories, under the test's own directory. */ + protected URI stageFile(URI testDir, String relativePath, String content) throws Exception { + try (BackupRepository repository = newRepository()) { + URI dir = testDir; + repository.createDirectory(dir); + final String[] parts = relativePath.split("/"); + for (int i = 0; i < parts.length - 1; i++) { + dir = repository.resolveDirectory(dir, parts[i]); + repository.createDirectory(dir); + } + final URI file = repository.resolve(dir, parts[parts.length - 1]); + try (OutputStream os = repository.createOutput(file)) { + os.write(content.getBytes(UTF_8)); + } + return file; + } + } + + /** A fresh, not yet created, directory in the repository for the current test to stage into. */ + protected URI newTestDir() throws Exception { + try (BackupRepository repository = newRepository()) { + return repository.resolveDirectory( + repository.createDirectoryURI(baseLocation()), UUID.randomUUID().toString()); + } + } + + private BackupRepository newRepository() { + return cluster.getJettySolrRunner(0).getCoreContainer().newBackupRepository(repositoryName()); + } + + private ConfigsetsApi.UploadConfigSet uploadRequest(String configSetName, URI source) { + final var request = new ConfigsetsApi.UploadConfigSet(configSetName, emptyBody()); + request.setRepository(repositoryName()); + request.setLocation(locationParam(source)); + return request; + } + + private ConfigsetsApi.UploadConfigSetFile uploadFileRequest( + String configSetName, String filePath, URI source) { + final var request = new ConfigsetsApi.UploadConfigSetFile(configSetName, filePath, emptyBody()); + request.setRepository(repositoryName()); + request.setLocation(locationParam(source)); + return request; + } + + private static InputStream emptyBody() { + return new ByteArrayInputStream(new byte[0]); + } + + private static String uniqueName(String prefix) { + return prefix + "-" + UUID.randomUUID(); + } + + private static boolean exists(String configSetName, String filePath) throws Exception { + return cluster.getZkClient().exists("/configs/" + configSetName + "/" + filePath); + } + + private static String read(String configSetName, String filePath) throws Exception { + return new String( + cluster.getZkClient().getData("/configs/" + configSetName + "/" + filePath, null, null), + UTF_8); + } + + @Test + public void testUploadConfigSetFromRepository() throws Exception { + final URI dir = newTestDir(); + stageFile(dir, "solrconfig.xml", ""); + stageFile(dir, "lang/stopwords.txt", "a"); + final String name = uniqueName("fromrepo"); + + uploadRequest(name, dir).process(cluster.getSolrClient()); + + assertEquals("", read(name, "solrconfig.xml")); + assertEquals("a", read(name, "lang/stopwords.txt")); + } + + @Test + public void testOverwriteFalseRejectsExistingConfigSet() throws Exception { + final URI dir = newTestDir(); + stageFile(dir, "solrconfig.xml", ""); + final String name = uniqueName("existing"); + uploadRequest(name, dir).process(cluster.getSolrClient()); + stageFile(dir, "solrconfig.xml", ""); + + final var request = uploadRequest(name, dir); + request.setOverwrite(false); + final var e = + expectThrows(RemoteSolrException.class, () -> request.process(cluster.getSolrClient())); + + assertEquals(400, e.code()); + assertEquals("", read(name, "solrconfig.xml")); + } + + @Test + public void testCleanupRemovesFilesNotInRepository() throws Exception { + final URI firstDir = newTestDir(); + stageFile(firstDir, "solrconfig.xml", ""); + stageFile(firstDir, "stale.txt", "stale"); + final String name = uniqueName("cleanup"); + uploadRequest(name, firstDir).process(cluster.getSolrClient()); + final URI secondDir = newTestDir(); + stageFile(secondDir, "solrconfig.xml", ""); + + final var request = uploadRequest(name, secondDir); + request.setCleanup(true); + request.process(cluster.getSolrClient()); + + assertEquals("", read(name, "solrconfig.xml")); + assertFalse(exists(name, "stale.txt")); + } + + @Test + public void testMissingLocationIsRejected() throws Exception { + final String name = uniqueName("missing"); + final var request = uploadRequest(name, newTestDir()); + + final var e = + expectThrows(RemoteSolrException.class, () -> request.process(cluster.getSolrClient())); + + assertEquals(400, e.code()); + assertFalse(cluster.getZkClient().exists("/configs/" + name)); + } + + @Test + public void testUploadSingleFileFromRepository() throws Exception { + final URI dir = newTestDir(); + final URI file = stageFile(dir, "files/schema.xml", ""); + final String name = uniqueName("singlefile"); + final URI baseDir = newTestDir(); + stageFile(baseDir, "solrconfig.xml", ""); + uploadRequest(name, baseDir).process(cluster.getSolrClient()); + + uploadFileRequest(name, "conf/schema.xml", file).process(cluster.getSolrClient()); + + assertEquals("", read(name, "conf/schema.xml")); + assertEquals("", read(name, "solrconfig.xml")); + } + + @Test + public void testSingleFileLocationMustBeAFile() throws Exception { + final URI dir = newTestDir(); + stageFile(dir, "files/schema.xml", ""); + final String name = uniqueName("notafile"); + final var request = uploadFileRequest(name, "schema.xml", newRepositoryChild(dir, "files")); + + final var e = + expectThrows(RemoteSolrException.class, () -> request.process(cluster.getSolrClient())); + + assertEquals(400, e.code()); + assertFalse(cluster.getZkClient().exists("/configs/" + name)); + } + + @Test + public void testRepositoryAndLocationAreBothRequired() throws Exception { + final var onlyRepository = new ConfigsetsApi.UploadConfigSet(uniqueName("c"), emptyBody()); + onlyRepository.setRepository(repositoryName()); + final var onlyLocation = new ConfigsetsApi.UploadConfigSet(uniqueName("c"), emptyBody()); + onlyLocation.setLocation(locationParam(newTestDir())); + + for (var request : new ConfigsetsApi.UploadConfigSet[] {onlyRepository, onlyLocation}) { + final var e = + expectThrows(RemoteSolrException.class, () -> request.process(cluster.getSolrClient())); + assertEquals(400, e.code()); + } + } + + @Test + public void testBodyAndRepositoryAreMutuallyExclusive() throws Exception { + final URI dir = newTestDir(); + stageFile(dir, "solrconfig.xml", ""); + final String name = uniqueName("both"); + final var request = + new ConfigsetsApi.UploadConfigSet( + name, new ByteArrayInputStream("not empty".getBytes(UTF_8))); + request.setRepository(repositoryName()); + request.setLocation(locationParam(dir)); + + final var e = + expectThrows(RemoteSolrException.class, () -> request.process(cluster.getSolrClient())); + + assertEquals(400, e.code()); + assertFalse(cluster.getZkClient().exists("/configs/" + name)); + } + + private URI newRepositoryChild(URI dir, String child) throws Exception { + try (BackupRepository repository = newRepository()) { + return repository.resolveDirectory(dir, child); + } + } +} From 7aeb2b691f69a9c4afff47fbf537daa3096ecad7 Mon Sep 17 00:00:00 2001 From: Jason Gerlowski Date: Mon, 9 Nov 2026 09:45:35 -0500 Subject: [PATCH 2/2] Minor doc tweak: v1/v2 example tabbing --- .../pages/configsets-api.adoc | 85 +++++++++++-------- 1 file changed, 50 insertions(+), 35 deletions(-) diff --git a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc index 302b4ac747fc..862ca6a79e6d 100644 --- a/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc +++ b/solr/solr-ref-guide/modules/configuration-guide/pages/configsets-api.adoc @@ -164,14 +164,56 @@ If set to `true`, Solr will overwrite an existing configset with the same name ( |=== When overwriting an existing configset (`overwrite=true`), this parameter tells Solr to delete the files in ZooKeeper that existed in the old configset but not in the one being uploaded. -When uploading an entire configset, the body of the request should be a zip file that contains the configset. -The zip file must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file). +Configsets can be uploadeded in two different ways: by providing compressed "zip" content in the request body, or by finding the configset using a "backup repository" (see below). +With the "zip" approach, the zip file/content must be created from within the `conf` directory (i.e., `solrconfig.xml` must be the top level entry in the zip file). + +Here is an example on how to create the zip file named "myconfig.zip" and upload it as a configset named "myConfigSet": + +[tabs#uploadconfigset] +====== +V1 API:: ++ +==== +With the v1 API, the `upload` command must be capitalized as `UPLOAD`: + +[source,bash] +---- +(cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip + +curl 'http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet' --header "Content-Type:application/octet-stream" --data-binary @myconfigset.zip +---- + +The same can be achieved using a Unix pipe with a single request as follows: + +[source,bash] +---- +(cd server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) | curl 'http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet' --header "Content-Type:application/octet-stream" --data-binary @- +---- +==== + +V2 API:: ++ +==== +With the v2 API, the name of the configset to upload is provided as a path parameter: + +[source,bash] +---- +(cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip + +curl -X PUT --header "Content-Type:application/octet-stream" --data-binary @myconfigset.zip + "http://localhost:8983/api/configsets/myConfigSet" +---- + +With this API, the default behavior is to overwrite the configset if it already exists. +This behavior can be disabled with the parameter `overwrite=false`, in which case the request will fail if the configset already exists. +==== +====== [[configsets-upload-repository]] === Upload from a Backup Repository -Instead of a zip file, the configset can be read from a directory in a xref:deployment-guide:backup-restore.adoc#backuprestore-storage-repositories[backup repository] (such as S3 or GCS), with `solrconfig.xml` at its top level. -Pass these parameters and no request body: +Configsets can also be uploaded to Solr using a xref:deployment-guide:backup-restore.adoc#backuprestore-storage-repositories[backup repository] (such as S3 or GCS), with `solrconfig.xml` at its top level. +Uploads using this "backup repository" approach must specify two additional parameters as coordinates for finding the configset files: `repository`:: + @@ -194,55 +236,28 @@ The directory in the repository that holds the configset's files. Required if `r Providing both a request body and `repository`/`location` is an error. The `overwrite` and `cleanup` parameters work the same way as for zip uploads. -[source,bash] ----- -curl -X PUT "http://localhost:8983/api/configsets/myConfigSet?repository=s3&location=/configsets/myconfig" ----- - -For the v1 API, use `/solr/admin/configs?action=UPLOAD&name=myConfigSet&repository=s3&location=/configsets/myconfig`. - -Here is an example on how to create the zip file named "myconfig.zip" and upload it as a configset named "myConfigSet": - -[tabs#uploadconfigset] +[tabs#uploadconfigsetrepo] ====== V1 API:: + ==== -With the v1 API, the `upload` command must be capitalized as `UPLOAD`: - [source,bash] ---- -(cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip - -curl 'http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet' --header "Content-Type:application/octet-stream" --data-binary @myconfigset.zip ----- - -The same can be achieved using a Unix pipe with a single request as follows: - -[source,bash] ----- -(cd server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) | curl 'http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet' --header "Content-Type:application/octet-stream" --data-binary @- +curl -X POST "http://localhost:8983/solr/admin/configs?action=UPLOAD&name=myConfigSet&repository=s3&location=/configsets/myconfig" ---- ==== V2 API:: + ==== -With the v2 API, the name of the configset to upload is provided as a path parameter: - [source,bash] ---- -(cd solr/server/solr/configsets/sample_techproducts_configs/conf && zip -r - *) > myconfigset.zip - -curl -X PUT --header "Content-Type:application/octet-stream" --data-binary @myconfigset.zip - "http://localhost:8983/api/configsets/myConfigSet" +curl -X PUT "http://localhost:8983/api/configsets/myConfigSet?repository=s3&location=/configsets/myconfig" ---- - -With this API, the default behavior is to overwrite the configset if it already exists. -This behavior can be disabled with the parameter `overwrite=false`, in which case the request will fail if the configset already exists. ==== ====== + [[configsets-upload-file]] == Update a Single File in a Configset