diff --git a/changelog/unreleased/SOLR-18457-migrate-update-api-to-jax-rs.yml b/changelog/unreleased/SOLR-18457-migrate-update-api-to-jax-rs.yml new file mode 100644 index 000000000000..a71b41df71fa --- /dev/null +++ b/changelog/unreleased/SOLR-18457-migrate-update-api-to-jax-rs.yml @@ -0,0 +1,14 @@ +title: > + Migrated the v2 update endpoints to JAX-RS. The v2 javabin endpoint is now /update/javabin, + where v1 uses the shorter /update/bin. Migrated the Admin UI's document-upload screen to use + these v2 endpoints. That screen now only supports the /update endpoint; it can no longer target + a custom endpoint. As part of updating the examples in the Ref Guide, the bundled techproducts + example schema also changed: "price" and "popularity" are now docValues-only (no longer indexed + or stored), the "price_c" copyField from "price" was removed, and nested/child-document support + ("_nest_path_"/"_nest_parent_") was added. +type: changed +authors: + - name: Eric Pugh +links: + - name: SOLR-18457 + url: https://issues.apache.org/jira/browse/SOLR-18457 diff --git a/solr/api/src/java/org/apache/solr/client/api/endpoint/UpdateApi.java b/solr/api/src/java/org/apache/solr/client/api/endpoint/UpdateApi.java new file mode 100644 index 000000000000..b798fc4ed1bd --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/endpoint/UpdateApi.java @@ -0,0 +1,211 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.endpoint; + +import static org.apache.solr.client.api.util.Constants.GENERIC_ENTITY_PROPERTY; +import static org.apache.solr.client.api.util.Constants.INDEX_PATH_PREFIX; + +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.Parameter; +import io.swagger.v3.oas.annotations.extensions.Extension; +import io.swagger.v3.oas.annotations.extensions.ExtensionProperty; +import io.swagger.v3.oas.annotations.parameters.RequestBody; +import jakarta.ws.rs.Consumes; +import jakarta.ws.rs.POST; +import jakarta.ws.rs.Path; +import jakarta.ws.rs.QueryParam; +import java.io.InputStream; +import org.apache.solr.client.api.model.UpdateResponse; +import org.apache.solr.client.api.util.StoreApiParameters; + +/** V2 API definitions for indexing documents via the update handler. */ +@Path(INDEX_PATH_PREFIX + "/update") +public interface UpdateApi { + + @POST + @Consumes({ + "application/json", + "text/json", + "application/xml", + "text/xml", + "application/csv", + "text/csv", + "application/javabin", + "application/cbor" + }) + @StoreApiParameters + @Operation( + summary = "Send updates using any supported content type", + tags = {"update"}) + UpdateResponse update( + @Parameter(description = "Commit the update immediately") @QueryParam("commit") + Boolean commit, + @Parameter(description = "Commit the update within this many milliseconds") + @QueryParam("commitWithin") + Integer commitWithin, + @Parameter(description = "Overwrite documents with the same unique key") + @QueryParam("overwrite") + Boolean overwrite, + @Parameter(description = "Perform a soft commit") @QueryParam("softCommit") + Boolean softCommit, + @Parameter(description = "Include assigned document versions in the response") + @QueryParam("versions") + Boolean versions, + @Parameter(required = true) + @RequestBody( + required = true, + description = "Update content in the format selected by the Content-Type header.", + extensions = + @Extension( + properties = { + @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value = "true") + })) + InputStream requestBody) + throws Exception; + + @POST + @Path("/json") + @Consumes({"application/json", "text/json"}) + @StoreApiParameters + @Operation( + summary = "Index documents in JSON format", + tags = {"update"}) + UpdateResponse updateJson( + @Parameter(description = "Commit the update immediately") @QueryParam("commit") + Boolean commit, + @Parameter(description = "Commit the update within this many milliseconds") + @QueryParam("commitWithin") + Integer commitWithin, + @Parameter(description = "Overwrite documents with the same unique key") + @QueryParam("overwrite") + Boolean overwrite, + @Parameter(description = "Perform a soft commit") @QueryParam("softCommit") + Boolean softCommit, + @Parameter(description = "Include assigned document versions in the response") + @QueryParam("versions") + Boolean versions, + @Parameter(required = true) + @RequestBody( + required = true, + description = "JSON update content.", + extensions = + @Extension( + properties = { + @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value = "true") + })) + InputStream requestBody) + throws Exception; + + @POST + @Path("/xml") + @Consumes({"application/xml", "text/xml"}) + @StoreApiParameters + @Operation( + summary = "Index documents in XML format", + tags = {"update"}) + UpdateResponse updateXml( + @Parameter(description = "Commit the update immediately") @QueryParam("commit") + Boolean commit, + @Parameter(description = "Commit the update within this many milliseconds") + @QueryParam("commitWithin") + Integer commitWithin, + @Parameter(description = "Overwrite documents with the same unique key") + @QueryParam("overwrite") + Boolean overwrite, + @Parameter(description = "Perform a soft commit") @QueryParam("softCommit") + Boolean softCommit, + @Parameter(description = "Include assigned document versions in the response") + @QueryParam("versions") + Boolean versions, + @Parameter(required = true) + @RequestBody( + required = true, + description = "XML update content.", + extensions = + @Extension( + properties = { + @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value = "true") + })) + InputStream requestBody) + throws Exception; + + @POST + @Path("/csv") + @Consumes({"application/csv", "text/csv"}) + @StoreApiParameters + @Operation( + summary = "Index documents in CSV format", + tags = {"update"}) + UpdateResponse updateCsv( + @Parameter(description = "Commit the update immediately") @QueryParam("commit") + Boolean commit, + @Parameter(description = "Commit the update within this many milliseconds") + @QueryParam("commitWithin") + Integer commitWithin, + @Parameter(description = "Overwrite documents with the same unique key") + @QueryParam("overwrite") + Boolean overwrite, + @Parameter(description = "Perform a soft commit") @QueryParam("softCommit") + Boolean softCommit, + @Parameter(description = "Include assigned document versions in the response") + @QueryParam("versions") + Boolean versions, + @Parameter(required = true) + @RequestBody( + required = true, + description = "CSV update content.", + extensions = + @Extension( + properties = { + @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value = "true") + })) + InputStream requestBody) + throws Exception; + + @POST + @Path("/javabin") + @Consumes("application/javabin") + @StoreApiParameters + @Operation( + summary = "Index documents in Javabin format", + tags = {"update"}) + UpdateResponse updateJavabin( + @Parameter(description = "Commit the update immediately") @QueryParam("commit") + Boolean commit, + @Parameter(description = "Commit the update within this many milliseconds") + @QueryParam("commitWithin") + Integer commitWithin, + @Parameter(description = "Overwrite documents with the same unique key") + @QueryParam("overwrite") + Boolean overwrite, + @Parameter(description = "Perform a soft commit") @QueryParam("softCommit") + Boolean softCommit, + @Parameter(description = "Include assigned document versions in the response") + @QueryParam("versions") + Boolean versions, + @Parameter(required = true) + @RequestBody( + required = true, + description = "Javabin update content.", + extensions = + @Extension( + properties = { + @ExtensionProperty(name = GENERIC_ENTITY_PROPERTY, value = "true") + })) + InputStream requestBody) + throws Exception; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/ToleratedUpdateError.java b/solr/api/src/java/org/apache/solr/client/api/model/ToleratedUpdateError.java new file mode 100644 index 000000000000..4437ca898187 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/ToleratedUpdateError.java @@ -0,0 +1,39 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** + * A single add/delete/delete-by-query that failed but was tolerated because the request's {@code + * maxErrors} allowed it. + */ +public class ToleratedUpdateError { + + @JsonProperty("type") + @Schema(description = "The kind of command that failed, e.g. 'ADD', 'DELID', or 'DELQ'.") + public String type; + + @JsonProperty("id") + @Schema(description = "The document id or delete-by-query expression that failed.") + public String id; + + @JsonProperty("message") + @Schema(description = "The error message describing why the command failed.") + public String message; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/UpdateResponse.java b/solr/api/src/java/org/apache/solr/client/api/model/UpdateResponse.java new file mode 100644 index 000000000000..3482a8b26571 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/UpdateResponse.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; +import java.util.List; + +/** Version information returned by an update request with {@code versions=true}. */ +public class UpdateResponse extends SolrJerseyResponse { + + @JsonProperty("adds") + @Schema(description = "Documents added and the versions assigned to them.") + public List adds; + + @JsonProperty("deletes") + @Schema(description = "Documents deleted and the versions assigned to their delete operations.") + public List deletes; + + @JsonProperty("deleteByQuery") + @Schema(description = "Delete-by-query operations and the versions assigned to them.") + public List deleteByQuery; + + @JsonProperty("errors") + @Schema( + description = + "Commands that failed but were tolerated because the request's maxErrors allowed it. " + + "Present only when the update chain includes a tolerant-update processor.") + public List errors; + + @JsonProperty("maxErrors") + @Schema( + description = + "The effective maxErrors value used for this request; -1 means unlimited. Present only " + + "when the update chain includes a tolerant-update processor.") + public Integer maxErrors; + + @JsonProperty("rf") + @Schema( + description = + "The replication factor actually achieved for this update. Present only in SolrCloud.") + public Integer rf; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/VersionedDocument.java b/solr/api/src/java/org/apache/solr/client/api/model/VersionedDocument.java new file mode 100644 index 000000000000..230dacda77bd --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/VersionedDocument.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** A document identifier and the version assigned to an update operation. */ +public class VersionedDocument { + + @JsonProperty("id") + @Schema(description = "The document's unique identifier.") + public String id; + + @JsonProperty("version") + @Schema(description = "The version assigned to the operation.") + public long version; +} diff --git a/solr/api/src/java/org/apache/solr/client/api/model/VersionedQuery.java b/solr/api/src/java/org/apache/solr/client/api/model/VersionedQuery.java new file mode 100644 index 000000000000..9eee66c69bd7 --- /dev/null +++ b/solr/api/src/java/org/apache/solr/client/api/model/VersionedQuery.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.client.api.model; + +import com.fasterxml.jackson.annotation.JsonProperty; +import io.swagger.v3.oas.annotations.media.Schema; + +/** A delete-by-query expression and the version assigned to the operation. */ +public class VersionedQuery { + + @JsonProperty("query") + @Schema(description = "The delete-by-query expression.") + public String query; + + @JsonProperty("version") + @Schema(description = "The version assigned to the operation.") + public long version; +} diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 8ec27435be5c..470b2f57ac01 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -346,7 +346,7 @@ private void createCollection(CloudSolrClient cloudSolrClient, CreateParams para + " using the Configsets V2 API"); var uploadReq = new ConfigsetsApi.UploadConfigSet( - confName, new ByteArrayInputStream(Utils.zipDirectory(confPath))); + confName, new ByteArrayInputStream(Utils.zipDirectory(confPath)), "application/zip"); uploadReq.process(cloudSolrClient); } diff --git a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java index f1383c784efb..826843d1a14d 100644 --- a/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java +++ b/solr/core/src/java/org/apache/solr/cli/packagemanager/PackageUtils.java @@ -91,7 +91,7 @@ public static ObjectMapper getMapper() { public static void postFile(SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { try (final var stream = new ByteBufferInputStream(buffer)) { - final var uploadReq = new FileStoreApi.UploadFile(name, stream); + final var uploadReq = new FileStoreApi.UploadFile(name, stream, "application/octet-stream"); if (sig != null) { uploadReq.setSig(List.of(sig)); } diff --git a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java index 5f59eae2f788..cbec302194ba 100644 --- a/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java +++ b/solr/core/src/java/org/apache/solr/handler/RequestHandlerBase.java @@ -229,6 +229,43 @@ public NamedList getInitArgs() { public abstract void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception; + /** + * Applies this handler's configured {@code defaults}/{@code appends}/{@code invariants} and + * HTTP-caching settings, then delegates to {@link #handleRequestBody}, handling {@link + * QueryLimitsExceededException} and any other exception the same way {@link #handleRequest} does + * -- but without touching this handler's own {@link HandlerMetrics} or timer. + * + *

Exceptions are recorded onto {@code rsp} via {@link SolrQueryResponse#setException} rather + * than thrown, matching {@link #handleRequest}'s contract. This exists for callers whose + * surrounding framework already records equivalent metrics itself (e.g. a JAX-RS resource wrapped + * by Jersey request/response filters) and would otherwise double them by also going through + * {@link #handleRequest}. + * + * @return true if {@link #handleRequestBody} returned normally; false if a {@link + * QueryLimitsExceededException} or other exception was caught and handled instead. {@link + * #handleRequest} uses this to decide whether to count a timeout, matching its behavior of + * only doing so on the normal path. + */ + public boolean handleRequestWithoutMetrics(SolrQueryRequest req, SolrQueryResponse rsp) { + try { + TestInjection.injectLeaderTragedy(req.getCore()); + if (pluginInfo != null && pluginInfo.attributes.containsKey(USEPARAM)) + req.getContext().put(USEPARAM, pluginInfo.attributes.get(USEPARAM)); + SolrPluginUtils.setDefaults(this, req, defaults, appends, invariants); + req.getContext().remove(USEPARAM); + rsp.setHttpCaching(httpCaching); + handleRequestBody(req, rsp); + return true; + } catch (QueryLimitsExceededException e) { + rsp.setPartialResults(req); + return false; + } catch (Exception e) { + Exception normalized = processReceivedException(req, e); + rsp.setException(normalized); + return false; + } + } + @Override public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { assert metrics != null @@ -242,25 +279,15 @@ public void handleRequest(SolrQueryRequest req, SolrQueryResponse rsp) { AttributedLongTimer.MetricTimer timer = metrics.requestTimes.start(); try { - TestInjection.injectLeaderTragedy(req.getCore()); - if (pluginInfo != null && pluginInfo.attributes.containsKey(USEPARAM)) - req.getContext().put(USEPARAM, pluginInfo.attributes.get(USEPARAM)); - SolrPluginUtils.setDefaults(this, req, defaults, appends, invariants); - req.getContext().remove(USEPARAM); - rsp.setHttpCaching(httpCaching); - handleRequestBody(req, rsp); - // count timeouts - - if (!haveCompleteResults(rsp.getResponseHeader())) { - metrics.numTimeouts.inc(); - rsp.setHttpCaching(false); + if (handleRequestWithoutMetrics(req, rsp)) { + // count timeouts + if (!haveCompleteResults(rsp.getResponseHeader())) { + metrics.numTimeouts.inc(); + rsp.setHttpCaching(false); + } + } else if (rsp.getException() != null) { + processErrorMetricsOnException(rsp.getException(), metrics); } - } catch (QueryLimitsExceededException e) { - rsp.setPartialResults(req); - } catch (Exception e) { - Exception normalized = processReceivedException(req, e); - processErrorMetricsOnException(normalized, metrics); - rsp.setException(normalized); } finally { try { timer.stop(); diff --git a/solr/core/src/java/org/apache/solr/handler/V2UpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/V2UpdateRequestHandler.java index d4bbae9b7b6e..2a55407d897f 100644 --- a/solr/core/src/java/org/apache/solr/handler/V2UpdateRequestHandler.java +++ b/solr/core/src/java/org/apache/solr/handler/V2UpdateRequestHandler.java @@ -18,26 +18,34 @@ package org.apache.solr.handler; import java.util.Collection; -import org.apache.solr.api.AnnotatedApi; +import java.util.List; import org.apache.solr.api.Api; +import org.apache.solr.api.JerseyResource; import org.apache.solr.handler.admin.api.UpdateAPI; +import org.apache.solr.jersey.APIConfigProvider; /** - * An extension of {@link UpdateRequestHandler} used solely to register the v2 /update APIs + * An extension of {@link UpdateRequestHandler} used solely to register the v2 /update APIs. * *

At core-load time, Solr looks at each 'plugin' in ImplicitPlugins.json, fetches the v2 {@link - * Api} implementations associated with each RequestHandler, and registers them in an {@link - * org.apache.solr.api.ApiBag}. Since UpdateRequestHandler is mentioned multiple times in - * ImplicitPlugins.json (once for each update API: /update, /update/json, etc.), this would cause + * org.apache.solr.api.Api} implementations associated with each RequestHandler, and registers them + * in an {@link org.apache.solr.api.ApiBag}. Since UpdateRequestHandler is mentioned multiple times + * in ImplicitPlugins.json (once for each update API: /update, /update/json, etc.), this would cause * the v2 APIs to be registered in duplicate. To avoid this, Solr has this RequestHandler, whose * only purpose is to register the v2 APIs that conceptually should be associated with * UpdateRequestHandler. */ -public class V2UpdateRequestHandler extends UpdateRequestHandler { +public class V2UpdateRequestHandler extends UpdateRequestHandler + implements APIConfigProvider { @Override public Collection getApis() { - return AnnotatedApi.getApis(new UpdateAPI(this)); + return List.of(); + } + + @Override + public Collection> getJerseyResources() { + return List.of(UpdateAPI.class); } @Override @@ -49,4 +57,14 @@ public Boolean registerV1() { public Boolean registerV2() { return Boolean.TRUE; } + + @Override + public UpdateAPI.UpdateRequestHandlerConfig provide() { + return new UpdateAPI.UpdateRequestHandlerConfig(this); + } + + @Override + public Class getConfigClass() { + return UpdateAPI.UpdateRequestHandlerConfig.class; + } } diff --git a/solr/core/src/java/org/apache/solr/handler/admin/api/UpdateAPI.java b/solr/core/src/java/org/apache/solr/handler/admin/api/UpdateAPI.java index e97198dec080..f8237a91821f 100644 --- a/solr/core/src/java/org/apache/solr/handler/admin/api/UpdateAPI.java +++ b/solr/core/src/java/org/apache/solr/handler/admin/api/UpdateAPI.java @@ -17,52 +17,219 @@ package org.apache.solr.handler.admin.api; -import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST; import static org.apache.solr.common.params.CommonParams.PATH; import static org.apache.solr.security.PermissionNameProvider.Name.UPDATE_PERM; -import org.apache.solr.api.EndPoint; +import jakarta.inject.Inject; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; +import org.apache.solr.api.JerseyResource; +import org.apache.solr.client.api.endpoint.UpdateApi; +import org.apache.solr.client.api.model.ToleratedUpdateError; +import org.apache.solr.client.api.model.UpdateResponse; +import org.apache.solr.client.api.model.VersionedDocument; +import org.apache.solr.client.api.model.VersionedQuery; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.core.SolrCore; import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.jersey.APIConfigProvider; +import org.apache.solr.jersey.PermissionName; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.response.SolrQueryResponse; /** - * All v2 APIs that share a prefix of /update + * V2 API implementation for indexing documents. * - *

Most of these v2 APIs are implemented as pure "pass-throughs" to the v1 code paths, but there - * are a few exceptions: /update and /update/json are both rewritten to /update/json/docs. + *

These APIs delegate to the v1 {@link UpdateRequestHandler}. Most are pure pass-throughs: + * {@code /update} (and {@code /update/xml}, {@code /update/csv}) select a loader by content-type, + * the same as v1's {@code /update}. The exceptions are {@code /update/json}, which is rewritten to + * {@code /update/json/docs} so a JSON body is processed as literal documents rather than + * update-command syntax, and {@code /update/javabin}, which is rewritten to {@code /update/bin} to + * select the javabin loader explicitly. */ -public class UpdateAPI { +public class UpdateAPI extends JerseyResource implements UpdateApi { + private final UpdateRequestHandler updateRequestHandler; + private final SolrQueryRequest solrQueryRequest; + private final SolrQueryResponse solrQueryResponse; + + @Inject + public UpdateAPI( + UpdateRequestHandlerConfig handlerConfig, + SolrQueryRequest solrQueryRequest, + SolrQueryResponse solrQueryResponse) { + this.updateRequestHandler = handlerConfig.updateRequestHandler; + this.solrQueryRequest = solrQueryRequest; + this.solrQueryResponse = solrQueryResponse; + } + + // Query parameters like commit, overwrite, etc are declared as method arguments for the + // JAX-RS/OpenAPI contract and via magic are read in by the handler. + @Override + @PermissionName(UPDATE_PERM) + public UpdateResponse update( + Boolean commit, + Integer commitWithin, + Boolean overwrite, + Boolean softCommit, + Boolean versions, + InputStream requestBody) + throws Exception { + return handleUpdate(null); + } - public UpdateAPI(UpdateRequestHandler updateRequestHandler) { - this.updateRequestHandler = updateRequestHandler; + @Override + @PermissionName(UPDATE_PERM) + public UpdateResponse updateJson( + Boolean commit, + Integer commitWithin, + Boolean overwrite, + Boolean softCommit, + Boolean versions, + InputStream requestBody) { + return handleUpdate(UpdateRequestHandler.DOC_PATH); } - @EndPoint(method = POST, path = "/update", permission = UPDATE_PERM) - public void update(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - req.getContext().put(PATH, "/update/json/docs"); - updateRequestHandler.handleRequest(req, rsp); + @Override + @PermissionName(UPDATE_PERM) + public UpdateResponse updateXml( + Boolean commit, + Integer commitWithin, + Boolean overwrite, + Boolean softCommit, + Boolean versions, + InputStream requestBody) { + return handleUpdate(null); } - @EndPoint(method = POST, path = "/update/xml", permission = UPDATE_PERM) - public void updateXml(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - updateRequestHandler.handleRequest(req, rsp); + @Override + @PermissionName(UPDATE_PERM) + public UpdateResponse updateCsv( + Boolean commit, + Integer commitWithin, + Boolean overwrite, + Boolean softCommit, + Boolean versions, + InputStream requestBody) { + return handleUpdate(null); } - @EndPoint(method = POST, path = "/update/csv", permission = UPDATE_PERM) - public void updateCsv(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - updateRequestHandler.handleRequest(req, rsp); + @Override + @PermissionName(UPDATE_PERM) + public UpdateResponse updateJavabin( + Boolean commit, + Integer commitWithin, + Boolean overwrite, + Boolean softCommit, + Boolean versions, + InputStream requestBody) { + return handleUpdate(UpdateRequestHandler.BIN_PATH); } - @EndPoint(method = POST, path = "/update/json", permission = UPDATE_PERM) - public void updateJson(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - req.getContext().put(PATH, "/update/json/docs"); - updateRequestHandler.handleRequest(req, rsp); + private UpdateResponse handleUpdate(String pathOverride) { + final UpdateResponse response = instantiateJerseyResponse(UpdateResponse.class); + if (pathOverride != null) { + solrQueryRequest.getContext().put(PATH, pathOverride); + } + // TolerantUpdateProcessor (errors/maxErrors) and DistributedZkUpdateProcessor (rf) write + // their payload into the legacy response header while handling the request. Initialize it + // for the handler, copy that payload onto the typed response, then discard the header so + // only one responseHeader (the typed Jersey one) is returned to the client. + // + // Uses handleRequestWithoutMetrics rather than handleRequest: this resource is wrapped by + // Jersey's RequestMetricHandling filters, which already record request/error metrics for + // updateRequestHandler (see PluginBag.JaxrsResourceToHandlerMappings) -- handleRequest would + // double-count both. + SolrCore.preDecorateResponse(solrQueryRequest, solrQueryResponse); + // UpdateRequestHandler.setDefaultWT() (called during the delegation below) mutates this same + // request's params to inject its own default 'wt' -- e.g. "xml" for the XML loader -- purely + // to pick a v1 response writer. That's a v1-internal implementation detail, but since it's + // the same SolrQueryRequest instance Jersey's MediaTypeOverridingFilter reads afterward, the + // injected value leaks out and hijacks this v2 response's actual Content-Type (which should + // stay JSON, or follow the client's real Accept header, regardless of which v1 loader ran). + // Restore the original params afterward so no such delegate-internal mutation survives. + final var originalParams = solrQueryRequest.getParams(); + try { + updateRequestHandler.handleRequestWithoutMetrics(solrQueryRequest, solrQueryResponse); + } finally { + solrQueryRequest.setParams(originalParams); + copyToleratedUpdateMetadata(response); + solrQueryResponse.getValues().remove("responseHeader"); + } + rethrowAnyException(solrQueryResponse); + response.adds = takeDocumentVersionResults("adds"); + response.deletes = takeDocumentVersionResults("deletes"); + response.deleteByQuery = takeQueryVersionResults(); + return response; } - @EndPoint(method = POST, path = "/update/bin", permission = UPDATE_PERM) - public void updateJavabin(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { - updateRequestHandler.handleRequest(req, rsp); + /** + * Copies replication-factor and tolerant-update-error metadata off the legacy response header and + * onto the typed response, before that header is discarded. + * + * @see org.apache.solr.update.processor.TolerantUpdateProcessor + * @see org.apache.solr.update.processor.DistributedZkUpdateProcessor + */ + @SuppressWarnings("unchecked") + private void copyToleratedUpdateMetadata(UpdateResponse response) { + final NamedList header = solrQueryResponse.getResponseHeader(); + if (header == null) return; + + final Object rf = header.get("rf"); + if (rf != null) response.rf = ((Number) rf).intValue(); + + final Object maxErrors = header.get("maxErrors"); + if (maxErrors != null) response.maxErrors = ((Number) maxErrors).intValue(); + + final var rawErrors = (List>) header.get("errors"); + if (rawErrors != null && !rawErrors.isEmpty()) { + response.errors = new ArrayList<>(rawErrors.size()); + for (NamedList rawError : rawErrors) { + final ToleratedUpdateError error = new ToleratedUpdateError(); + error.type = rawError.get("type"); + error.id = rawError.get("id"); + error.message = rawError.get("message"); + response.errors.add(error); + } + } } + + private List takeDocumentVersionResults(String name) { + final NamedList values = (NamedList) solrQueryResponse.getValues().remove(name); + if (values == null) return null; + final List results = new ArrayList<>(values.size()); + for (int i = 0; i < values.size(); i++) { + final VersionedDocument result = new VersionedDocument(); + result.id = values.getName(i); + result.version = ((Number) values.getVal(i)).longValue(); + results.add(result); + } + return results; + } + + private List takeQueryVersionResults() { + final NamedList values = + (NamedList) solrQueryResponse.getValues().remove("deleteByQuery"); + if (values == null) return null; + final List results = new ArrayList<>(values.size()); + for (int i = 0; i < values.size(); i++) { + final VersionedQuery result = new VersionedQuery(); + result.query = values.getName(i); + result.version = ((Number) values.getVal(i)).longValue(); + results.add(result); + } + return results; + } + + private void rethrowAnyException(SolrQueryResponse rsp) { + final Exception ex = rsp.getException(); + if (ex instanceof SolrException solrEx) throw solrEx; + if (ex != null) throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, ex); + } + + /** Configuration object providing access to the {@link UpdateRequestHandler} instance. */ + public record UpdateRequestHandlerConfig(UpdateRequestHandler updateRequestHandler) + implements APIConfigProvider.APIConfig {} } diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java index f4378a96a0a4..38c9cb9864e7 100644 --- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java +++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java @@ -34,6 +34,7 @@ import java.util.List; import java.util.Locale; import java.util.Map; +import java.util.Set; import java.util.UUID; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.solr.common.SolrException; @@ -120,6 +121,51 @@ private static boolean mapEntryIsChildDoc(Object val) { return val instanceof Map; } + // Field-name keys recognized by AtomicUpdateDocumentMerger when it merges a doc containing + // atomic-update instructions (e.g. {"add": "foo"}) into an existing one. + private static final Set ATOMIC_UPDATE_OPERATIONS = + Set.of("add", "set", "remove", "removeregex", "inc", "add-distinct"); + + /** + * Guards the literal-document (docs/split-mode) path against silently swallowing atomic-update + * syntax: that path always overwrites the document with exactly the literal fields it's given, so + * a value shaped like {@code {"add": "foo"}} would otherwise be indexed as a nested child + * document (or, under the default field mapping, flattened into a field literally named e.g. + * "cat.add"), discarding whatever the rest of the target document held, with no error to say so. + * Depending on the request's split/f params, by the time a record reaches here such a value has + * either survived as a nested {@code Map}/{@code List}, or already been flattened by {@link + * org.apache.solr.common.util.JsonRecordReader} into a dotted field name whose last segment is + * the operator name -- this checks for both shapes. + */ + private static void rejectAtomicUpdateSyntax(Map doc) { + for (Map.Entry e : doc.entrySet()) { + String key = e.getKey(); + int lastDot = key.lastIndexOf('.'); + boolean flattenedOperation = + lastDot >= 0 && ATOMIC_UPDATE_OPERATIONS.contains(key.substring(lastDot + 1)); + if (flattenedOperation || looksLikeAtomicUpdateOperation(e.getValue())) { + throw new SolrException( + SolrException.ErrorCode.BAD_REQUEST, + "Field '" + + (lastDot >= 0 ? key.substring(0, lastDot) : key) + + "' looks like an atomic update operation (e.g. '{\"add\": ...}'), but this " + + "endpoint indexes documents literally and does not support atomic updates; use " + + UpdateRequestHandler.JSON_PATH + + " instead."); + } + } + } + + private static boolean looksLikeAtomicUpdateOperation(Object val) { + if (val instanceof Map map) { + return map.keySet().stream().anyMatch(ATOMIC_UPDATE_OPERATIONS::contains); + } + if (val instanceof List list) { + return list.stream().anyMatch(JsonLoader::looksLikeAtomicUpdateOperation); + } + return false; + } + static class SingleThreadedJsonLoader extends ContentStreamLoader { protected final UpdateRequestProcessor processor; @@ -277,6 +323,7 @@ public void handle(Map record, String path) { changeChildDoc(copy); docs.add(copy); } else { + rejectAtomicUpdateSyntax(copy); AddUpdateCommand cmd = new AddUpdateCommand(req); cmd.commitWithin = commitWithin; cmd.overwrite = overwrite; diff --git a/solr/core/src/java/org/apache/solr/jersey/container/ContainerRequestUtils.java b/solr/core/src/java/org/apache/solr/jersey/container/ContainerRequestUtils.java index f3371c8f0e6c..e96433c0814b 100644 --- a/solr/core/src/java/org/apache/solr/jersey/container/ContainerRequestUtils.java +++ b/solr/core/src/java/org/apache/solr/jersey/container/ContainerRequestUtils.java @@ -124,6 +124,10 @@ private static URI getRequestUri(HttpServletRequest httpServletRequest, URI base // Jersey is only used for v2 APIs so we have no need of the janky v2 suffixing (and it impedes // matching) - remove if present. uri = uri.replace("/solr/____v2", ""); + // Normalize the /c/ alias to /collections/ so JAX-RS path matching works correctly. + // V2HttpCall already resolves the collection from /c/ paths, but Jersey sees the original URI + // and its path regex only matches 'cores' or 'collections', not the 'c' shorthand. + uri = normalizeCAlias(uri); final String queryString = httpServletRequest.getQueryString(); if (queryString != null) { @@ -140,4 +144,20 @@ private static String getServerAddress(URI baseUri) { } return serverAddress; } + + /** + * Normalizes the {@code /c/} collection alias to {@code /collections/} so that JAX-RS path + * matching works correctly. The V2 API supports {@code /c/} as a shorthand for {@code + * /collections/}, but JAX-RS path templates use the regex {@code cores|collections} to match the + * index-type segment. + */ + static String normalizeCAlias(String uri) { + if (uri.startsWith("/c/")) { + return "/collections/" + uri.substring(3); + } + if (uri.equals("/c")) { + return "/collections"; + } + return uri; + } } diff --git a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java index f2f80ed019e1..860a148db5f2 100644 --- a/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java +++ b/solr/core/src/test/org/apache/solr/cloud/TestConfigSetsAPI.java @@ -1219,7 +1219,7 @@ private int uploadGivenConfigSet( try (InputStream fileBytes = new FileInputStream(file.toFile())) { // Create and execute the upload request using generated client final var uploadRequest = - new ConfigsetsApi.UploadConfigSet(configSetName + suffix, fileBytes); + new ConfigsetsApi.UploadConfigSet(configSetName + suffix, fileBytes, "application/zip"); uploadRequest.setOverwrite(overwrite); uploadRequest.setCleanup(cleanup); @@ -1280,7 +1280,7 @@ private int uploadSingleConfigSetFile( // Create and execute the upload request using generated client final var uploadRequest = new ConfigsetsApi.UploadConfigSetFile( - configSetName + suffix, uploadPath, sampleConfigFile); + configSetName + suffix, uploadPath, sampleConfigFile, "application/octet-stream"); final var response = uploadRequest.process(cluster.getSolrClient()); return response.responseHeader.status; diff --git a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java index 466e99844496..90ceb81d32b9 100644 --- a/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java +++ b/solr/core/src/test/org/apache/solr/filestore/TestDistribFileStore.java @@ -348,7 +348,9 @@ public static void uploadKey(byte[] bytes, String path, MiniSolrCloudCluster clu public static UploadToFileStoreResponse postFile( SolrClient client, ByteBuffer buffer, String name, String sig) throws SolrServerException, IOException { - final var uploadReq = new FileStoreApi.UploadFile(name, new ByteBufferInputStream(buffer)); + final var uploadReq = + new FileStoreApi.UploadFile( + name, new ByteBufferInputStream(buffer), "application/octet-stream"); if (sig != null) { uploadReq.setSig(List.of(sig)); } diff --git a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java index 1ae42e0cf229..02957ce1481e 100644 --- a/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java +++ b/solr/core/src/test/org/apache/solr/handler/JsonLoaderTest.java @@ -329,6 +329,54 @@ public void testJsonDocFormat() throws Exception { p.addCommands.get(0).solrDoc.toString()); } + @Test + public void testDocsPathRejectsFlattenedAtomicUpdateFieldName() throws Exception { + // This endpoint indexes documents literally; unlike testAtomicUpdateFieldValue's use of the + // regular /update/json add-command path, a value shaped like {"add": "foo"} here would + // otherwise silently flatten (under the default field mapping) into a field literally named + // "cat.add", fully overwriting the target document rather than adding to its "cat" field. + String doc = "{\"id\": \"1\", \"cat\": {\"add\": \"y\"}}"; + SolrQueryRequest req = req(); + req.getContext().put("path", "/update/json/docs"); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader(); + + SolrException ex = + expectThrows( + SolrException.class, + () -> loader.load(req, rsp, new ContentStreamBase.StringStream(doc), p)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue(ex.getMessage().contains("cat")); + assertTrue(ex.getMessage().contains("atomic update")); + assertEquals(0, p.addCommands.size()); + + req.close(); + } + + @Test + public void testDocsPathRejectsAtomicUpdateSyntaxPreservedByCustomSplit() throws Exception { + // With a split config that targets "cat" as its own record boundary, the atomic-update-shaped + // value survives as a literal Map (rather than being flattened, as in the test above) - this + // exercises that second shape of the same guard. + String doc = "{\"id\": \"1\", \"cat\": [{\"add\": \"y\"}]}"; + SolrQueryRequest req = req("split", "/|/cat"); + req.getContext().put("path", "/update/json/docs"); + SolrQueryResponse rsp = new SolrQueryResponse(); + BufferingRequestProcessor p = new BufferingRequestProcessor(null); + JsonLoader loader = new JsonLoader(); + + SolrException ex = + expectThrows( + SolrException.class, + () -> loader.load(req, rsp, new ContentStreamBase.StringStream(doc), p)); + assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, ex.code()); + assertTrue(ex.getMessage().contains("cat")); + assertEquals(0, p.addCommands.size()); + + req.close(); + } + public void testSrcAndUniqueDocs() throws Exception { BufferingRequestProcessor p; JsonLoader loader; diff --git a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java index fe430c5907d8..4b5bb59f6676 100644 --- a/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java +++ b/solr/core/src/test/org/apache/solr/handler/RequestHandlerBaseTest.java @@ -25,6 +25,7 @@ import static org.mockito.Mockito.never; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; import static org.mockito.Mockito.when; import io.opentelemetry.api.common.AttributeKey; @@ -36,12 +37,18 @@ import org.apache.solr.common.SolrException; import org.apache.solr.common.params.MapSolrParams; import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrCore; import org.apache.solr.metrics.SolrMetricsContext; +import org.apache.solr.metrics.otel.instruments.AttributedLongTimer; import org.apache.solr.request.SolrQueryRequest; import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.search.QueryLimitsExceededException; import org.apache.solr.search.SyntaxError; +import org.apache.solr.security.AuthorizationContext; +import org.apache.solr.security.PermissionNameProvider; import org.junit.Before; import org.junit.BeforeClass; import org.junit.Test; @@ -200,9 +207,126 @@ public CoreContainer getCoreContainer() { assertTrue(RequestHandlerBase.isInternalShardRequest(solrQueryRequest)); } + @Test + public void testHandleRequestWithoutMetricsTouchesNoMetrics() { + final RequestHandlerBase.HandlerMetrics metrics = createHandlerMetrics(); + final TestHandler handler = new TestHandler(metrics, (req, rsp) -> {}); + final SolrQueryResponse rsp = new SolrQueryResponse(); + + final boolean completedNormally = handler.handleRequestWithoutMetrics(testRequest(), rsp); + + assertTrue(completedNormally); + verifyNoInteractions(mockLongCounter, mockLongHistogram); + } + + @Test + public void testHandleRequestCountsExactlyOneRequestOnSuccess() { + // A caller relying on handleRequest (rather than handleRequestWithoutMetrics, as UpdateAPI + // now does) should still see exactly the pre-existing metrics behavior: one counter touch + // (requests.inc()) for a request that completes normally with no timeout. + final RequestHandlerBase.HandlerMetrics metrics = createHandlerMetrics(); + final TestHandler handler = new TestHandler(metrics, (req, rsp) -> {}); + final SolrQueryResponse rsp = new SolrQueryResponse(); + rsp.addResponseHeader(new SimpleOrderedMap<>()); + + handler.handleRequest(testRequest(), rsp); + + verify(mockLongCounter, times(1)).add(eq(1L), any()); + } + + @Test + public void testHandleRequestDoesNotCountQueryLimitsExceededAsATimeout() { + // rsp.setPartialResults(req) (called for QueryLimitsExceededException) makes + // haveCompleteResults(...) false, same as an actual timeout would -- but this path must not + // also increment numTimeouts, matching pre-existing behavior of only doing so when + // handleRequestBody returns normally. If it were double-counted, this would see 2 calls + // instead of 1. + final RequestHandlerBase.HandlerMetrics metrics = createHandlerMetrics(); + final TestHandler handler = + new TestHandler( + metrics, + (req, rsp) -> { + throw new QueryLimitsExceededException("over limit"); + }); + final SolrQueryResponse rsp = new SolrQueryResponse(); + rsp.addResponseHeader(new SimpleOrderedMap<>()); + + handler.handleRequest(testRequest(), rsp); + + verify(mockLongCounter, times(1)).add(eq(1L), any()); + } + + @Test + public void testHandleRequestCountsRequestAndErrorOnException() { + final RequestHandlerBase.HandlerMetrics metrics = createHandlerMetrics(); + final TestHandler handler = + new TestHandler( + metrics, + (req, rsp) -> { + throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "boom"); + }); + final SolrQueryResponse rsp = new SolrQueryResponse(); + rsp.addResponseHeader(new SimpleOrderedMap<>()); + + handler.handleRequest(testRequest(), rsp); + + // requests.inc() (top of handleRequest) + processErrorMetricsOnException's client-error + // increment: exactly 2 calls total, not 3 (which double-accounting would produce). + verify(mockLongCounter, times(2)).add(eq(1L), any()); + verify(mockLongCounter, times(1)) + .add(eq(1L), argThat(attrs -> "client".equals(attrs.get(SOURCE_ATTR)))); + } + + private SolrQueryRequest testRequest() { + return new SolrQueryRequestBase(solrCore, new ModifiableSolrParams()) { + @Override + public CoreContainer getCoreContainer() { + return coreContainer; + } + }; + } + + /** Minimal concrete {@link RequestHandlerBase} whose body and metrics are test-controlled. */ + private static class TestHandler extends RequestHandlerBase { + @FunctionalInterface + interface Body { + void run(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception; + } + + private final Body body; + + TestHandler(RequestHandlerBase.HandlerMetrics metrics, Body body) { + this.metrics = metrics; + this.body = body; + // createHandlerMetrics() only mocks the 2-arg longHistogram(name, description) overload; + // AttributedInstrumentFactory.attributedLongTimer actually calls the 3-arg overload, so + // metrics.requestTimes ends up backed by a real (unmocked, null-histogram) instance. Swap + // it for a no-op stand-in so handleRequest()'s timer.start()/stop() don't NPE. + metrics.requestTimes = + new AttributedLongTimer(null, null) { + @Override + public void record(Long value) {} + }; + } + + @Override + public void handleRequestBody(SolrQueryRequest req, SolrQueryResponse rsp) throws Exception { + body.run(req, rsp); + } + + @Override + public String getDescription() { + return "test"; + } + + @Override + public PermissionNameProvider.Name getPermissionName(AuthorizationContext ctx) { + return null; + } + } + // Ideally we wouldn't need to use mocks here, but HandlerMetrics requires a SolrMetricsContext, - // which - // requires a MetricsManager, which requires ... + // which requires a MetricsManager, which requires ... private RequestHandlerBase.HandlerMetrics createHandlerMetrics() { final SolrMetricsContext metricsContext = mock(SolrMetricsContext.class); diff --git a/solr/core/src/test/org/apache/solr/handler/V2UpdateAPIMappingTest.java b/solr/core/src/test/org/apache/solr/handler/V2UpdateAPIMappingTest.java deleted file mode 100644 index 24dadde25109..000000000000 --- a/solr/core/src/test/org/apache/solr/handler/V2UpdateAPIMappingTest.java +++ /dev/null @@ -1,117 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.solr.handler; - -import static org.apache.solr.common.params.CommonParams.PATH; -import static org.mockito.Mockito.mock; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import org.apache.solr.SolrTestCaseJ4; -import org.apache.solr.api.Api; -import org.apache.solr.api.ApiBag; -import org.apache.solr.common.params.ModifiableSolrParams; -import org.apache.solr.common.util.CommandOperation; -import org.apache.solr.handler.admin.api.UpdateAPI; -import org.apache.solr.request.SolrQueryRequest; -import org.apache.solr.request.SolrQueryRequestBase; -import org.apache.solr.response.SolrQueryResponse; -import org.junit.Before; -import org.junit.BeforeClass; -import org.junit.Test; - -/** Unit tests for the v2 to v1 mapping logic in {@link UpdateAPI} */ -public class V2UpdateAPIMappingTest extends SolrTestCaseJ4 { - private ApiBag apiBag; - - @BeforeClass - public static void ensureWorkingMockito() { - assumeWorkingMockito(); - } - - @Before - public void setupApiBag() { - UpdateRequestHandler mockUpdateHandler = mock(UpdateRequestHandler.class); - apiBag = new ApiBag(false); - final UpdateAPI updateAPI = new UpdateAPI(mockUpdateHandler); - - apiBag.registerObject(updateAPI); - } - - @Test - public void testUpdateApiRewriting() { - - // Assume JSON in path if no specific format specified - { - final SolrQueryRequest req = runUpdateApi("/update"); - assertEquals("/update/json/docs", req.getContext().get(PATH)); - } - - // Rewrite v2 /update/json to v1's /update/json/docs - { - final SolrQueryRequest req = runUpdateApi("/update/json"); - assertEquals("/update/json/docs", req.getContext().get(PATH)); - } - - // No rewriting for /update/xml, /update/csv, or /update/bin - { - final SolrQueryRequest req = runUpdateApi("/update/xml"); - assertEquals("/update/xml", req.getContext().get(PATH)); - } - { - final SolrQueryRequest req = runUpdateApi("/update/csv"); - assertEquals("/update/csv", req.getContext().get(PATH)); - } - { - final SolrQueryRequest req = runUpdateApi("/update/bin"); - assertEquals("/update/bin", req.getContext().get(PATH)); - } - } - - private SolrQueryRequest runUpdateApi(String path) { - final HashMap parts = new HashMap<>(); - final Api api = apiBag.lookup(path, "POST", parts); - final SolrQueryResponse rsp = new SolrQueryResponse(); - final SolrQueryRequestBase req = new SolrQueryRequestBase(null, new ModifiableSolrParams()); - req.getContext().put(PATH, path); - - api.call(req, rsp); - - return req; - } - - private SolrQueryRequestBase createTestRequest(Map pathTemplateValues) { - return new SolrQueryRequestBase(null, new ModifiableSolrParams()) { - @Override - public List getCommands(boolean validateInput) { - return List.of(); - } - - @Override - public Map getPathTemplateValues() { - return pathTemplateValues; - } - - @Override - public String getHttpMethod() { - return "POST"; - } - }; - } -} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIMetricsDoubleCountingTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIMetricsDoubleCountingTest.java new file mode 100644 index 000000000000..c3ec270d2245 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIMetricsDoubleCountingTest.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.common.SolrException; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.handler.RequestHandlerBase; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.metrics.otel.instruments.AttributedLongCounter; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.apache.solr.util.ContentStreamBase; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Covers a bug flagged in review of SOLR-18457 (PR #4177): {@link UpdateAPI#handleUpdate} used to + * call {@code updateRequestHandler.handleRequest(...)} -- the full v1 {@link + * org.apache.solr.handler.RequestHandlerBase#handleRequest} wrapper, not just the inner + * request-processing logic. That wrapper does its own metrics bookkeeping on the handler's {@link + * org.apache.solr.handler.RequestHandlerBase.HandlerMetrics}: it increments {@code requests}, times + * the call, and on failure increments {@code numClientErrors}/{@code numServerErrors}. + * + *

But {@code updateRequestHandler} here is the very same {@link + * org.apache.solr.handler.V2UpdateRequestHandler} instance that Jersey's {@code + * RequestMetricHandling.PreRequestMetricsFilter}/{@code PostRequestMetricsFilter} already use to + * record metrics for this same request (via {@code PluginBag.JaxrsResourceToHandlerMappings}, keyed + * by the {@link UpdateAPI} resource class) -- and, on failure, {@code + * UpdateAPI#rethrowAnyException} re-throws so the same exception reaches {@code + * CatchAllExceptionMapper}, which recorded the error a second time too. So every v2 update request + * used to be counted twice, and every v2 update error used to be counted twice. + * + *

The fix: {@link UpdateAPI#handleUpdate} now calls the new {@link + * RequestHandlerBase#handleRequestWithoutMetrics}, which runs everything {@code handleRequest} does + * (configured defaults/appends/invariants, HTTP caching, exception handling) except the + * metrics/timer bookkeeping, leaving that solely to the surrounding Jersey filters. + * + *

This test doesn't stand up a real Jersey container; it fetches the real, core-registered + * handler (so metrics are genuinely initialized) and simulates the surrounding filters/mapper + * directly, to isolate the accounting to {@code UpdateAPI.handleUpdate} rather than the + * request-processing logic alone. + */ +public class UpdateAPIMetricsDoubleCountingTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + @Test + public void testSuccessfulRequestCountedOnce() throws Exception { + final UpdateRequestHandler handler = updateHandler(); + final RequestHandlerBase.HandlerMetrics metrics = handler.getMetricsForThisRequest(null); + final AtomicLong requestCount = new AtomicLong(); + metrics.requests = countingCounter(requestCount); + + try (SolrQueryRequestBase req = jsonRequest("[{\"id\":\"1\"}]")) { + final SolrQueryResponse rsp = new SolrQueryResponse(); + final UpdateAPI api = + new UpdateAPI(new UpdateAPI.UpdateRequestHandlerConfig(handler), req, rsp); + + // Simulates RequestMetricHandling.PreRequestMetricsFilter, which increments this same + // handler's metrics before the Jersey resource method (UpdateAPI.update) ever runs. + metrics.requests.inc(); + + api.update(null, null, null, null, null, null); + + assertEquals( + "one logical v2 update request should count as exactly one request -- " + + "UpdateAPI must not also increment PreRequestMetricsFilter's HandlerMetrics via " + + "handleRequest()", + 1L, + requestCount.get()); + } + } + + @Test + public void testFailedRequestErrorCountedOnce() throws Exception { + final UpdateRequestHandler handler = updateHandler(); + final RequestHandlerBase.HandlerMetrics metrics = handler.getMetricsForThisRequest(null); + final AtomicLong clientErrorCount = new AtomicLong(); + metrics.numClientErrors = countingCounter(clientErrorCount); + + // No "id": AddUpdateCommand rejects this with a 400 (client error) once it reaches + // RunUpdateProcessorFactory, deep inside updateRequestHandler.handleRequest(). + try (SolrQueryRequestBase req = jsonRequest("[{\"title\":\"no id here\"}]")) { + final SolrQueryResponse rsp = new SolrQueryResponse(); + final UpdateAPI api = + new UpdateAPI(new UpdateAPI.UpdateRequestHandlerConfig(handler), req, rsp); + + final SolrException thrown = + expectThrows(SolrException.class, () -> api.update(null, null, null, null, null, null)); + + // UpdateAPI re-throws so that CatchAllExceptionMapper can build the error response and + // record the error metric -- simulating that here, since there's no real Jersey container + // in this test. handleRequestWithoutMetrics itself must not have already recorded it too. + RequestHandlerBase.processErrorMetricsOnException(thrown, metrics); + + assertEquals( + "one failed v2 update request should count as exactly one client error -- " + + "UpdateAPI must not record error metrics itself now that it uses " + + "handleRequestWithoutMetrics, leaving that solely to CatchAllExceptionMapper", + 1L, + clientErrorCount.get()); + } + } + + private static UpdateRequestHandler updateHandler() { + // V2UpdateRequestHandler.registerV1() is false, so it's deliberately absent from the v1 + // handler registry core.getRequestHandler(...) reads from -- it's only reachable via the + // same JAX-RS resource-to-handler mapping RequestMetricHandling.PreRequestMetricsFilter uses. + final UpdateRequestHandler handler = + (UpdateRequestHandler) + h.getCore().getRequestHandlers().getJaxrsRegistry().get(UpdateAPI.class); + assertNotNull( + "expected the v2-only V2UpdateRequestHandler to be registered for the UpdateAPI resource", + handler); + return handler; + } + + private static SolrQueryRequestBase jsonRequest(String json) { + final SolrQueryRequestBase req = new SolrQueryRequestBase(h.getCore(), SolrParams.of()); + req.setContentStreams(List.of(new ContentStreamBase.StringStream(json, "application/json"))); + return req; + } + + /** An {@link AttributedLongCounter} that tallies into {@code count} instead of touching OTel. */ + private static AttributedLongCounter countingCounter(AtomicLong count) { + return new AttributedLongCounter(null, null) { + @Override + public void add(Long value) { + count.addAndGet(value); + } + }; + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIRequestParamLeakTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIRequestParamLeakTest.java new file mode 100644 index 000000000000..1cf6969e8227 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIRequestParamLeakTest.java @@ -0,0 +1,95 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.util.Map; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.UpdateResponse; +import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.params.MapSolrParams; +import org.apache.solr.common.params.SolrParams; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.request.SolrQueryRequestBase; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Covers a bug flagged in review of SOLR-18457 (PR #4177): {@code + * UpdateRequestHandler.setDefaultWT()} mutates the shared {@link SolrQueryRequest} to inject its + * own default {@code wt} (e.g. {@code "xml"}, picked by {@code XMLLoader}) purely to select a v1 + * response writer. Since {@link UpdateAPI#handleUpdate} hands that same request object to Jersey's + * {@code MediaTypeOverridingFilter} afterward, the injected value used to leak out and hijack the + * v2 response's actual {@code Content-Type} -- overriding whatever the client's real {@code Accept} + * header asked for, regardless of which v1 loader happened to run internally. In the Admin UI's + * document-upload screen (Solr XML entry mode), this meant the generated JS client received a + * response it couldn't recognize as JSON and showed nothing, even though indexing itself had + * succeeded. + * + *

This test doesn't exercise the real {@code XMLLoader} (that's covered elsewhere); it fakes the + * handler to mutate the request's params exactly the way {@code setDefaultWT} does, isolating the + * fix to {@link UpdateAPI#handleUpdate} restoring the request's original params afterward. + */ +public class UpdateAPIRequestParamLeakTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + @Test + public void testDelegateInjectedWtDoesNotSurviveTheRequest() throws Exception { + final SolrQueryRequest req = new SolrQueryRequestBase(h.getCore(), SolrParams.of()); + final SolrQueryResponse rsp = new SolrQueryResponse(); + final UpdateAPI api = + new UpdateAPI( + new UpdateAPI.UpdateRequestHandlerConfig(new WtInjectingUpdateRequestHandler()), + req, + rsp); + + try { + assertNull( + "sanity check: the incoming request shouldn't already have a wt param", + req.getParams().get(CommonParams.WT)); + + final UpdateResponse response = api.update(null, null, null, null, null, null); + assertNotNull(response); + + assertNull( + "UpdateRequestHandler.setDefaultWT()'s internal default must not survive past " + + "handleUpdate() -- it would hijack MediaTypeOverridingFilter's response " + + "Content-Type decision for a request that never asked for 'wt' at all", + req.getParams().get(CommonParams.WT)); + } finally { + req.close(); + } + } + + /** + * Mutates the request's params exactly the way {@code UpdateRequestHandler.setDefaultWT} does. + */ + private static class WtInjectingUpdateRequestHandler extends UpdateRequestHandler { + @Override + public boolean handleRequestWithoutMetrics(SolrQueryRequest req, SolrQueryResponse rsp) { + final SolrParams params = req.getParams(); + req.setParams( + SolrParams.wrapDefaults(params, new MapSolrParams(Map.of(CommonParams.WT, "xml")))); + return true; + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIResponseHeaderTest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIResponseHeaderTest.java new file mode 100644 index 000000000000..d15f2aac4ab3 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPIResponseHeaderTest.java @@ -0,0 +1,104 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.handler.admin.api; + +import java.util.List; +import org.apache.solr.SolrTestCaseJ4; +import org.apache.solr.client.api.model.ToleratedUpdateError; +import org.apache.solr.client.api.model.UpdateResponse; +import org.apache.solr.common.util.SimpleOrderedMap; +import org.apache.solr.handler.UpdateRequestHandler; +import org.apache.solr.request.SolrQueryRequest; +import org.apache.solr.response.SolrQueryResponse; +import org.junit.BeforeClass; +import org.junit.Test; + +/** + * Covers a bug flagged in review of SOLR-18457 (PR #4177): {@link UpdateAPI#handleUpdate} used to + * unconditionally remove the entire legacy {@code responseHeader} after delegating to {@link + * UpdateRequestHandler}, rather than just the duplicate {@code status}/{@code QTime} it was written + * to suppress. Two real processors write payload into that header while handling a request -- + * {@code TolerantUpdateProcessor} ({@code errors}/{@code maxErrors}) and {@code + * DistributedZkUpdateProcessor} ({@code rf}, the achieved replication factor) -- so {@link + * UpdateAPI#handleUpdate} now copies that payload onto {@link UpdateResponse} before discarding the + * header. + * + *

This test doesn't exercise the real processors (that's TolerantUpdateProcessorTest's job); it + * fakes the handler to write exactly what they would, isolating the fix to {@code + * UpdateAPI.handleUpdate} itself. + */ +public class UpdateAPIResponseHeaderTest extends SolrTestCaseJ4 { + + @BeforeClass + public static void beforeClass() throws Exception { + initCore("solrconfig.xml", "schema.xml"); + } + + @Test + public void testTolerantErrorsAndReplicationFactorSurviveResponseHeaderRemoval() + throws Exception { + final SolrQueryRequest req = req(); + final SolrQueryResponse rsp = new SolrQueryResponse(); + final UpdateAPI api = + new UpdateAPI( + new UpdateAPI.UpdateRequestHandlerConfig(new HeaderStuffingUpdateRequestHandler()), + req, + rsp); + + try { + final UpdateResponse response = api.update(null, null, null, null, null, null); + + assertNull( + "the legacy responseHeader itself should still be gone -- its payload should have " + + "been copied onto the typed response instead, not left behind", + rsp.getValues().get("responseHeader")); + + assertEquals(Integer.valueOf(1), response.rf); + assertEquals(Integer.valueOf(10), response.maxErrors); + assertNotNull(response.errors); + assertEquals(1, response.errors.size()); + final ToleratedUpdateError error = response.errors.get(0); + assertEquals("ADD", error.type); + assertEquals("bad-doc-1", error.id); + assertEquals("ERROR: failed to add doc", error.message); + } finally { + req.close(); + } + } + + /** + * Simulates what {@code TolerantUpdateProcessor.finish()} (errors/maxErrors) and {@code + * DistributedZkUpdateProcessor} (rf) do to a real request's response header, without needing a + * real tolerant-update chain or SolrCloud cluster. + */ + private static class HeaderStuffingUpdateRequestHandler extends UpdateRequestHandler { + @Override + public boolean handleRequestWithoutMetrics(SolrQueryRequest req, SolrQueryResponse rsp) { + // TolerantUpdateProcessor.java:276,279 + final SimpleOrderedMap toleratedError = new SimpleOrderedMap<>(); + toleratedError.add("type", "ADD"); + toleratedError.add("id", "bad-doc-1"); + toleratedError.add("message", "ERROR: failed to add doc"); + rsp.getResponseHeader().add("errors", List.of(toleratedError)); + rsp.getResponseHeader().add("maxErrors", 10); + + // DistributedZkUpdateProcessor.java:1391 + rsp.getResponseHeader().add("rf", 1); + return true; + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPITest.java b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPITest.java new file mode 100644 index 000000000000..72523cd6a727 --- /dev/null +++ b/solr/core/src/test/org/apache/solr/handler/admin/api/UpdateAPITest.java @@ -0,0 +1,397 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.solr.handler.admin.api; + +import static org.apache.solr.core.CoreContainer.ALLOW_PATHS_SYSPROP; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import org.apache.solr.SolrTestCase; +import org.apache.solr.client.api.model.IndexType; +import org.apache.solr.client.solrj.SolrClient; +import org.apache.solr.client.solrj.SolrRequest; +import org.apache.solr.client.solrj.request.GenericSolrRequest; +import org.apache.solr.client.solrj.request.GenericV2SolrRequest; +import org.apache.solr.client.solrj.request.JavaBinUpdateRequestCodec; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.RequestWriter; +import org.apache.solr.client.solrj.request.UpdateApi; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.response.JavaBinResponseParser; +import org.apache.solr.client.solrj.response.QueryResponse; +import org.apache.solr.client.solrj.response.XMLResponseParser; +import org.apache.solr.client.solrj.response.json.JsonMapResponseParser; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.ModifiableSolrParams; +import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.NamedList; +import org.apache.solr.util.ExternalPaths; +import org.apache.solr.util.SolrJettyTestRule; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; + +/** + * Integration tests for the v2 update API endpoints implemented via JAX-RS in {@link UpdateAPI}. + */ +public class UpdateAPITest extends SolrTestCase { + + @ClassRule public static SolrJettyTestRule solrTestRule = new SolrJettyTestRule(); + + private static final String CORE_NAME = "update-api-test"; + private static final String CUSTOM_JSON_CORE_NAME = "custom-json-update-api-test"; + + @BeforeClass + public static void beforeClass() throws Exception { + EnvUtils.setProperty( + ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); + solrTestRule.startSolr(createTempDir()); + solrTestRule + .newCollection(CORE_NAME) + .withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET) + .create(); + // can't use techproducts config because it enables srcField, which is incompatible with nested + // split=/exams requests + solrTestRule + .newCollection(CUSTOM_JSON_CORE_NAME) + .withConfigSet(ExternalPaths.DEFAULT_CONFIGSET) + .create(); + } + + @Test + public void testV1AndV2GenericUpdateParityAcrossFormats() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + + for (UpdateFormat format : UpdateFormat.values()) { + final String v1Id = "parity-v1-" + format.name().toLowerCase(Locale.ROOT); + final String v2Id = "parity-v2-" + format.name().toLowerCase(Locale.ROOT); + + final NamedList v1Response = sendV1Update(client, format, v1Id); + final NamedList v2Response = sendV2Update(client, format, v2Id); + + assertLegacySuccessfulAdd(format, v1Id, v1Response); + assertTypedSuccessfulAdd(format, v2Id, v2Response); + assertIndexed(client, v1Id); + assertIndexed(client, v2Id); + } + } + + @Test + public void testV1AndV2CustomJsonTransformParity() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CUSTOM_JSON_CORE_NAME); + final String payload = "{\"exams\":[{\"id\":\"custom-json-v1\",\"name\":\"V1 document\"}]}"; + + final ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("split", "/exams"); + params.add("f", "id:/exams/id"); + params.add("f", "name_s:/exams/name"); + params.set("commit", true); + + final GenericSolrRequest v1Request = + new GenericSolrRequest(SolrRequest.METHOD.POST, "/update/json/docs", params); + v1Request.setRequiresCollection(true); + v1Request.setContentWriter( + new RequestWriter.StringPayloadContentWriter(payload, "application/json")); + client.request(v1Request, CUSTOM_JSON_CORE_NAME); + + final String v2Payload = payload.replace("custom-json-v1", "custom-json-v2"); + final GenericV2SolrRequest v2Request = + new GenericV2SolrRequest( + SolrRequest.METHOD.POST, "/cores/" + CUSTOM_JSON_CORE_NAME + "/update/json", params); + v2Request.setContentWriter( + new RequestWriter.StringPayloadContentWriter(v2Payload, "application/json")); + client.request(v2Request); + + assertIndexedField(client, CUSTOM_JSON_CORE_NAME, "custom-json-v1", "name_s", "V1 document"); + assertIndexedField(client, CUSTOM_JSON_CORE_NAME, "custom-json-v2", "name_s", "V1 document"); + } + + @Test + public void testUpdateJsonViaV2Api() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + + // POST via the V2 /update/json endpoint (also rewrites to /update/json/docs) + final GenericV2SolrRequest addReq = + new GenericV2SolrRequest(SolrRequest.METHOD.POST, "/cores/" + CORE_NAME + "/update/json"); + addReq.setContentWriter( + new RequestWriter.StringPayloadContentWriter( + "[{\"id\":\"v2updatejson1\",\"title\":\"V2 update/json test\"}]", "application/json")); + client.request(addReq); + + // Commit via standard SolrJ commit: /update/json is rewritten to /update/json/docs (see + // UpdateAPI's javadoc), so unlike bare /update it doesn't support command syntax like + // {"commit":{}}. + client.commit(CORE_NAME); + + // Verify + final ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set("q", "id:v2updatejson1"); + final QueryResponse queryRsp = new QueryRequest(queryParams).process(client, CORE_NAME); + assertEquals(1, queryRsp.getResults().getNumFound()); + } + + @Test + public void testGeneratedUpdateApiClientSetsAMatchingContentType() throws Exception { + // The generated SolrJ client used to hardcode Content-Type: application/octet-stream for + // this request body, which none of /update's @Consumes media types accept -- every call + // failed with a 415. It must now let the caller supply a content type the server accepts. + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + final String json = "[{\"id\":\"v2-generated-client-update1\"}]"; + final var request = + new UpdateApi.Update( + IndexType.CORE, + CORE_NAME, + new ByteArrayInputStream(json.getBytes(StandardCharsets.UTF_8)), + "application/json"); + + request.process(client); + client.commit(CORE_NAME); + + assertIndexed(client, "v2-generated-client-update1"); + } + + @Test + public void testUpdateXmlViaV2Api() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + + // POST via the V2 /update/xml endpoint + final GenericV2SolrRequest addReq = + new GenericV2SolrRequest(SolrRequest.METHOD.POST, "/cores/" + CORE_NAME + "/update/xml"); + addReq.setContentWriter( + new RequestWriter.StringPayloadContentWriter( + "v2updatexml1" + + "V2 update/xml test", + "application/xml")); + client.request(addReq); + + // Commit via standard SolrJ commit: /update/xml negotiates on Content-Type like v1's /update + // and does support XML command syntax (e.g. ) -- kept separate here just for parity + // with the other format tests in this file. + client.commit(CORE_NAME); + + // Verify + final ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set("q", "id:v2updatexml1"); + final QueryResponse queryRsp = new QueryRequest(queryParams).process(client, CORE_NAME); + assertEquals(1, queryRsp.getResults().getNumFound()); + } + + @Test + public void testUpdateJavabinViaV2Api() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + final SolrInputDocument doc = new SolrInputDocument(); + doc.setField("id", "v2updatejavabin1"); + final UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.add(doc); + updateRequest.deleteById("v2deleteversion1"); + updateRequest.deleteByQuery("id:v2deletequery1"); + final ByteArrayOutputStream payload = new ByteArrayOutputStream(); + new JavaBinUpdateRequestCodec().marshal(updateRequest, payload); + + final ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("versions", "true"); + final GenericV2SolrRequest addReq = + new GenericV2SolrRequest( + SolrRequest.METHOD.POST, "/cores/" + CORE_NAME + "/update/javabin", params); + addReq.setResponseParser(new JavaBinResponseParser()); + addReq.withContent(payload.toByteArray(), "application/javabin"); + final NamedList updateResponse = client.request(addReq); + assertEquals(1, updateResponse.getAll("responseHeader").size()); + assertTypedVersion(updateResponse, "adds", "id", "v2updatejavabin1", true); + assertTypedVersion(updateResponse, "deletes", "id", "v2deleteversion1", false); + assertTypedVersion(updateResponse, "deleteByQuery", "query", "id:v2deletequery1", false); + client.commit(CORE_NAME); + + final ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set("q", "id:v2updatejavabin1"); + final QueryResponse queryRsp = new QueryRequest(queryParams).process(client, CORE_NAME); + assertEquals(1, queryRsp.getResults().getNumFound()); + } + + @Test + public void testUpdateReturnsAssignedVersion() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + final ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("versions", "true"); + final GenericV2SolrRequest addReq = + new GenericV2SolrRequest( + SolrRequest.METHOD.POST, "/cores/" + CORE_NAME + "/update", params); + addReq.setResponseParser(new JsonMapResponseParser()); + addReq.setContentWriter( + new RequestWriter.StringPayloadContentWriter( + "[{\"id\":\"v2version1\",\"name\":\"Generic V2 update document\"}]", + "application/json")); + + final var response = client.request(addReq); + assertTypedVersion(response, "adds", "id", "v2version1", true); + client.commit(CORE_NAME); + assertIndexedField(client, CORE_NAME, "v2version1", "name", "Generic V2 update document"); + } + + @Test + public void testXmlUpdateResponseHasOneHeaderAndVersion() throws Exception { + final SolrClient client = solrTestRule.getSolrClient(CORE_NAME); + final ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("versions", "true"); + final GenericV2SolrRequest addReq = + new GenericV2SolrRequest( + SolrRequest.METHOD.POST, "/cores/" + CORE_NAME + "/update/xml", params); + addReq.setResponseParser(new XMLResponseParser()); + addReq.setContentWriter( + new RequestWriter.StringPayloadContentWriter( + "v2xmlversion1", "application/xml")); + + final NamedList response = client.request(addReq); + assertEquals(1, response.getAll("responseHeader").size()); + assertTypedVersion(response, "adds", "id", "v2xmlversion1", true); + } + + private static NamedList sendV1Update(SolrClient client, UpdateFormat format, String id) + throws Exception { + final GenericSolrRequest request = + new GenericSolrRequest(SolrRequest.METHOD.POST, "/update", updateParams()); + request.setRequiresCollection(true); + request.setResponseParser(new JsonMapResponseParser()); + request.withContent(format.payload(id), format.contentType); + return client.request(request, CORE_NAME); + } + + private static NamedList sendV2Update(SolrClient client, UpdateFormat format, String id) + throws Exception { + final GenericV2SolrRequest request = + new GenericV2SolrRequest( + SolrRequest.METHOD.POST, "/cores/" + CORE_NAME + "/update", updateParams()); + request.setResponseParser(new JsonMapResponseParser()); + request.withContent(format.payload(id), format.contentType); + return client.request(request); + } + + private static ModifiableSolrParams updateParams() { + final ModifiableSolrParams params = new ModifiableSolrParams(); + params.set("versions", true); + params.set("commit", true); + return params; + } + + private static void assertLegacySuccessfulAdd( + UpdateFormat format, String expectedId, NamedList response) { + assertEquals(format.name(), 1, response.getAll("responseHeader").size()); + final List adds = (List) response.get("adds"); + assertNotNull(format.name(), adds); + assertEquals(format.name(), expectedId, adds.get(0)); + assertTrue(format.name(), ((Number) adds.get(1)).longValue() > 0); + } + + private static void assertTypedSuccessfulAdd( + UpdateFormat format, String expectedId, NamedList response) { + assertEquals(format.name(), 1, response.getAll("responseHeader").size()); + assertTypedVersion(response, "adds", "id", expectedId, true); + } + + private static void assertTypedVersion( + NamedList response, + String field, + String key, + String expectedValue, + boolean positiveVersion) { + final List values = (List) response.get(field); + assertNotNull(field, values); + assertEquals(1, values.size()); + final Object value = values.get(0); + final Object actualValue; + final Object version; + if (value instanceof Map map) { + actualValue = map.get(key); + version = map.get("version"); + } else { + final NamedList namedValue = (NamedList) value; + actualValue = namedValue.get(key); + version = namedValue.get("version"); + } + assertEquals(expectedValue, actualValue); + final long numericVersion = ((Number) version).longValue(); + assertTrue(positiveVersion ? numericVersion > 0 : numericVersion < 0); + } + + private static void assertIndexed(SolrClient client, String id) throws Exception { + final ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set("q", "id:" + id); + final QueryResponse queryResponse = new QueryRequest(queryParams).process(client, CORE_NAME); + assertEquals(id, 1, queryResponse.getResults().getNumFound()); + } + + private static void assertIndexedField( + SolrClient client, String collection, String id, String field, String value) + throws Exception { + final ModifiableSolrParams queryParams = new ModifiableSolrParams(); + queryParams.set("q", "id:" + id); + queryParams.set("fl", field); + final QueryResponse queryResponse = new QueryRequest(queryParams).process(client, collection); + assertEquals(1, queryResponse.getResults().getNumFound()); + assertEquals(value, queryResponse.getResults().get(0).getFieldValue(field)); + } + + private enum UpdateFormat { + JSON("application/json") { + @Override + byte[] payload(String id) { + return bytes("{\"add\":{\"doc\":{\"id\":\"" + id + "\"}}}"); + } + }, + XML("application/xml") { + @Override + byte[] payload(String id) { + return bytes("" + id + ""); + } + }, + CSV("application/csv") { + @Override + byte[] payload(String id) { + return bytes("id\n" + id + "\n"); + } + }, + JAVABIN("application/javabin") { + @Override + byte[] payload(String id) throws Exception { + final SolrInputDocument doc = new SolrInputDocument(); + doc.setField("id", id); + final UpdateRequest updateRequest = new UpdateRequest(); + updateRequest.add(doc); + final ByteArrayOutputStream payload = new ByteArrayOutputStream(); + new JavaBinUpdateRequestCodec().marshal(updateRequest, payload); + return payload.toByteArray(); + } + }; + + private final String contentType; + + UpdateFormat(String contentType) { + this.contentType = contentType; + } + + abstract byte[] payload(String id) throws Exception; + + static byte[] bytes(String value) { + return value.getBytes(StandardCharsets.UTF_8); + } + } +} diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSolrJ.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSolrJ.java index 9d3a4fe13841..3508ce9f5f4b 100644 --- a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSolrJ.java +++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSolrJ.java @@ -149,7 +149,8 @@ public void testUpdateFileContentsBodyBinding() throws Exception { byte[] invalidXml = "".getBytes(StandardCharsets.UTF_8); var req = - new SchemaDesignerApi.UpdateFileContents(configSet, new ByteArrayInputStream(invalidXml)); + new SchemaDesignerApi.UpdateFileContents( + configSet, new ByteArrayInputStream(invalidXml), "application/xml"); req.setFile("solrconfig.xml"); SchemaDesignerFileContentsResponse resp = req.process(cluster.getSolrClient()); diff --git a/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java index f5854b6e296f..6f31447646a7 100644 --- a/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java +++ b/solr/core/src/test/org/apache/solr/pkg/ConfigsetPinnedPackageVersionTest.java @@ -129,7 +129,9 @@ private static String signature(byte[] content) throws Exception { private void uploadPluginJar(String version, Path jarPath) throws Exception { var pluginRequest = new FileStoreApi.UploadFile( - "/my-plugin/plugin-" + version + ".jar", Files.newInputStream(jarPath)); + "/my-plugin/plugin-" + version + ".jar", + Files.newInputStream(jarPath), + "application/octet-stream"); pluginRequest.setSig(List.of(signature(Files.readAllBytes(jarPath)))); pluginRequest.process(client); } diff --git a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema.xml b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema.xml index 4ff07b39c211..6b30a658c463 100644 --- a/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema.xml +++ b/solr/server/solr/configsets/sample_techproducts_configs/conf/managed-schema.xml @@ -121,6 +121,12 @@ --> + + + + + + - - @@ -286,6 +289,7 @@ + diff --git a/solr/solr-ref-guide/modules/indexing-guide/examples/IndexingNestedDocuments.java b/solr/solr-ref-guide/modules/indexing-guide/examples/IndexingNestedDocuments.java index 668abf3d5924..9139edf726e4 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/examples/IndexingNestedDocuments.java +++ b/solr/solr-ref-guide/modules/indexing-guide/examples/IndexingNestedDocuments.java @@ -23,6 +23,7 @@ import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.SolrQuery; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.common.SolrDocument; import org.apache.solr.common.SolrDocumentList; @@ -72,6 +73,10 @@ public void testIndexingAnonKids() throws Exception { CollectionAdminRequest.createCollection(collection, ANON_KIDS_CONFIG, 1, 1) .process(cluster.getSolrClient()); + // This example demonstrates anonymous children, which require a root-only schema. + new SchemaRequest.DeleteField("_nest_path_").process(cluster.getSolrClient(), collection); + new SchemaRequest.DeleteField("_nest_parent_").process(cluster.getSolrClient(), collection); + // configure the client with the default collection name, to simplify our example below. IndexingNestedDocuments.clientUsedInSolrJExample = cluster.newSolrClient(collection); diff --git a/solr/solr-ref-guide/modules/indexing-guide/images/documents-screen/documents_add_screen.png b/solr/solr-ref-guide/modules/indexing-guide/images/documents-screen/documents_add_screen.png index bbb5eb7c4b33..2c805618ebe9 100644 Binary files a/solr/solr-ref-guide/modules/indexing-guide/images/documents-screen/documents_add_screen.png and b/solr/solr-ref-guide/modules/indexing-guide/images/documents-screen/documents_add_screen.png differ diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/documents-screen.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/documents-screen.adoc index 13c96ecc5b95..470cb7995ff7 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/documents-screen.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/documents-screen.adoc @@ -36,9 +36,8 @@ There are other ways to load data, see also these sections: ==== == Common Fields -* Request-Handler: The first step is to define the RequestHandler. -By default `/update` will be defined. -Change the request handler to `/update/extract` to use Solr Cell, if you are running a Tika Server. +* Endpoint: The Documents screen submits updates to the v2 `/api/cores/\{core\}/update` or +`/api/collections/\{collection\}/update` endpoint, depending on whether Solr is running in standalone or SolrCloud mode. * Document Type: Select the Document Type to define the format of the document to load. The remaining parameters may change depending on the document type selected. * Document(s): Enter a properly formatted Solr document corresponding to the `Document Type` selected. @@ -69,16 +68,16 @@ The Document Builder provides a wizard-like interface to enter fields of a docum == File Upload The File Upload option allows choosing a prepared file and uploading it. -If using `/update` for the Request-Handler option, you will be limited to XML, CSV, and JSON. +The File Upload option currently uses the v1 `/update` handler and supports XML, CSV, and JSON files. Other document types (e.g., Word, PDF, etc.) can be indexed using the ExtractingRequestHandler (aka, Solr Cell). -You must modify the RequestHandler to `/update/extract`, which must be defined in your `solrconfig.xml` file with your desired defaults. +Solr Cell uploads through `/update/extract` must be sent directly to that configured request handler. You should also add `&literal.id` shown in the "Extracting Request Handler Params" field so the file chosen is given a unique id. More information can be found in xref:indexing-with-tika.adoc[]. == Solr Command -The Solr Command option allows you to use the `/update` request handler with XML or JSON formatted commands to perform specific actions. +The Solr Command option allows you to use the v2 `/update` endpoint with JSON formatted commands to perform specific actions. A few examples are: * Deleting documents diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc index c864a62fe8d8..2558bd243093 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/indexing-with-update-handlers.adoc @@ -20,23 +20,11 @@ Update handlers are request handlers designed to add, delete and update documents to the index. In addition to having plugins for importing rich documents (see xref:indexing-with-tika.adoc[]), Solr natively supports indexing structured documents in JSON, XML, and CSV. -The recommended way to configure and use request handlers is with path based names that map to paths in the request URL. -However, request handlers can also be specified with the `qt` (query type) parameter if the xref:configuration-guide:requestdispatcher.adoc[`requestDispatcher`] is appropriately configured. -It is possible to access the same handler using more than one name, which can be useful if you wish to specify different sets of default options. - A single unified update request handler supports JSON, XML, CSV, and javabin update requests, delegating to the appropriate `ContentStreamLoader` based on the `Content-Type` of the xref:content-streams.adoc[ContentStream]. If you need to pre-process documents after they are loaded but before they are indexed (or even checked against the schema), Solr has document preprocessing plugins for Update Request Handlers, called xref:configuration-guide:update-request-processors.adoc[], which allow for default and custom configuration chains. -== UpdateRequestHandler Configuration - -The default configuration file has the update request handler configured by default. - -[source,xml] ----- - ----- == JSON Formatted Index Updates @@ -56,7 +44,14 @@ To differentiate this from a set of commands, the `json.command=false` request p ==== Adding a Single JSON Document -The simplest way to add documents via JSON is to send each document individually as a JSON Object, using the `/update/json/docs` path: +The simplest way to add documents via JSON is to send each document individually as a JSON Object. + +[tabs#json-single-document-request] +====== +V1 API:: ++ +==== +The V1 document-only endpoint is `/update/json/docs`: [source,bash] ---- @@ -66,11 +61,33 @@ curl 'http://localhost:8983/solr/{collectionName}/update/json/docs' -H 'Content- "title": "Doc 1" }' ---- +==== + +V2 API:: ++ +==== +The V2 document-only endpoint is `/update/json`: + +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/json' -H 'Content-Type: application/json' --data-binary ' +{ + "id": "1", + "title": "Doc 1" +}' +---- +==== +====== ==== Adding Multiple JSON Documents Adding multiple documents at one time via JSON can be done via a JSON Array of JSON Objects, where each object represents a document: +[tabs#json-multiple-document-request] +====== +V1 API:: ++ +==== [source,bash] ---- curl 'http://localhost:8983/solr/{collectionName}/update' -H 'Content-Type: application/json' --data-binary ' @@ -85,19 +102,61 @@ curl 'http://localhost:8983/solr/{collectionName}/update' -H 'Content-Type: appl } ]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update' -H 'Content-Type: application/json' --data-binary ' +[ + { + "id": "1", + "title": "Doc 1" + }, + { + "id": "2", + "title": "Doc 2" + } +]' +---- +==== +====== -A sample JSON file is provided at `example/exampledocs/books.json` and contains an array of objects that you can add to the Solr "techproducts" example: +A JSON file can be submitted in the same way: +[tabs#json-file-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-type:application/json' --data-binary @example/exampledocs/books.json +---- +==== + +V2 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update?commit=true' --data-binary @example/exampledocs/books.json -H 'Content-type:application/json' +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-type:application/json' --data-binary @example/exampledocs/books.json ---- +==== +====== ==== Sending JSON Update Commands In general, the JSON update syntax supports all of the update commands that the XML update handler supports, through a straightforward mapping. Multiple commands, adding and deleting documents, may be contained in one message: +[tabs#json-command-request] +====== +V1 API:: ++ +==== [source,bash,subs="verbatim,callouts"] ---- curl 'http://localhost:8983/solr/{collectionName}/update' -H 'Content-Type: application/json' --data-binary ' @@ -125,7 +184,40 @@ curl 'http://localhost:8983/solr/{collectionName}/update' -H 'Content-Type: appl "delete": { "query":"QUERY" } --<6> }' ---- +==== +V2 API:: ++ +==== +[source,bash,subs="verbatim,callouts"] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update' -H 'Content-Type: application/json' --data-binary ' +{ + "add": { + "doc": { + "id": "DOC1", + "my_field": 2.3, + "my_multivalued_field": [ "aaa", "bbb" ] --<1> + } + }, + "add": { + "commitWithin": 5000, --<2> + "overwrite": false, --<3> + "doc": { + "f1": "v1", --<4> + "f1": "v2" + } + }, + + "commit": {}, + "optimize": { "waitSearcher":false }, + + "delete": { "id":"ID" }, --<5> + "delete": { "query":"QUERY" } --<6> +}' +---- +==== +====== <1> Can use an array for a multi-valued field <2> Commit this document within 5 seconds <3> Don't check for existing documents with the same uniqueKey @@ -167,27 +259,21 @@ You can specify the version of deletes in the body of the update request as well === JSON Update Convenience Paths -In addition to the `/update` handler, there are a few additional JSON specific request handler paths available by default in Solr, that implicitly override the behavior of some request parameters: +The V1 `/update/json` endpoint is a specialized Solr request path designed for clients that cannot easily set the required `Content-Type: application/json` header, as this route automatically forces Solr to treat the incoming payload as JSON. +The V1 `/update/json/docs` path additionally forces document-only processing through `stream.contentType=application/json` and `json.command=false`. -[width="100%",options="header",] -|=== -|Path |Default Parameters -|`/update/json` |`update.contentType=application/json` -|`/update/json/docs` a| -`stream.contentType=application/json` - -`json.command=false` - -|=== +In V2, use the general `/api/collections/\{collection\}/update` or `/api/cores/\{core\}/update` endpoint with `Content-Type: application/json` for JSON update commands. +Use `/api/collections/\{collection\}/update/json` or `/api/cores/\{core\}/update/json` for document-only JSON requests. -The `/update/json` path may be useful for clients sending in JSON formatted update commands from applications where setting the Content-Type proves difficult, while the `/update/json/docs` path can be particularly convenient for clients that always want to send in documents – either individually or as a list – without needing to worry about the full JSON command syntax. +WARNING: The document-only paths (`/update/json/docs` in V1, `/update/json` in V2) index every document literally and do not support xref:partial-document-updates.adoc#atomic-updates[atomic update] syntax -- use `/update` for that instead. +A field value shaped like an atomic-update operator, e.g. `{"add": "foo"}`, is rejected with an error rather than applied. +A document sent to these paths without atomic-update syntax, however, still fully replaces any existing document with the same `id`, the same as sending a plain document to `/update`. === Custom JSON Documents Solr can support custom JSON. This is covered in the section xref:transforming-and-indexing-custom-json.adoc[]. - == XML Formatted Index Updates Index update commands can be sent as XML message to the update handler using `Content-type: application/xml` or `Content-type: text/xml`. @@ -223,92 +309,41 @@ For example: ---- -The add command supports some optional attributes which may be specified. - -`commitWithin`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: none -|=== -+ -Add the document within the specified number of milliseconds. - -`overwrite`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: `true` -|=== -+ -Indicates if the unique key constraints should be checked to overwrite previous versions of the same document (see below). - -If the document schema defines a unique key, then by default an `/update` operation to add a document will overwrite (i.e., replace) any document in the index with the same unique key. -If no unique key has been defined, indexing performance is somewhat faster, as no check has to be made for an existing documents to replace. - -If you have a unique key field, but you feel confident that you can safely bypass the uniqueness check (e.g., you build your indexes in batch, and your indexing code guarantees it never adds the same document more than once) you can specify the `overwrite="false"` option when adding your documents. +The add command supports the shared `commitWithin` and `overwrite` options described in the <> section. +The XML syntax places these options on the `` element. === XML Update Commands -==== Commit and Optimize During Updates - -The `` operation writes all documents loaded since the last commit to one or more segment files on the disk. -Before a commit has been issued, newly indexed content is not visible to searches. -The commit operation opens a new searcher, and triggers any event listeners that have been configured. - -Commits may be issued explicitly with a `` message, and can also be triggered from `` parameters in `solrconfig.xml`. - -The `` operation requests Solr to merge internal data structures. -For a large index, optimization will take some time to complete, but by merging many small segment files into larger segments, search performance may improve. -If you are using Solr's replication mechanism to distribute searches across many systems, be aware that after an optimize, a complete index will need to be transferred. +The XML update syntax supports commit and optimize commands. Their behavior and options are described in the <> section above. -WARNING: You should only consider using optimize on static indexes, i.e., indexes that can be optimized as part of the regular update process (say once-a-day updates). -Applications requiring NRT functionality should not use optimize. - -The `` and `` elements accept these optional attributes: - -`waitSearcher`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: `true` -|=== -+ -Blocks until a new searcher is opened and registered as the main query searcher, making the changes visible. +Here are examples of `` and `` using optional attributes: -`expungeDeletes`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: `false` -|=== +[tabs#xml-commit-request] +====== +V1 API:: + -Merges segments that have more than 10% deleted docs, expunging the deleted documents in the process. -Resulting segments will respect `maxMergedSegmentMB`. -This option only applies in a `` operation. -+ -WARNING: `expungeDeletes` is less expensive than optimize, but the same warnings apply. +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary ' + + +' +---- +==== -`maxSegments`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: none -|=== +V2 API:: + -Makes a best effort attempt to merge the segments down to no more than this number of segments but does not guarantee that the goal will be achieved. -Unless there is tangible evidence that optimizing to a small number of segments is beneficial, this parameter should be omitted and the default behavior accepted. -This option only applies in a `` operation. -Default is unlimited, resulting segments respect the `maxMergedSegmentMB` setting. - -Here are examples of `` and `` using optional attributes: - -[source,xml] +==== +[source,bash] ---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/xml' -H "Content-Type: text/xml" --data-binary ' - +' ---- +==== +====== ==== Delete Operations @@ -318,15 +353,38 @@ It doesn't work for child/nested docs. "Delete by Query" deletes all documents matching a specified query, although `commitWithin` is ignored for a Delete by Query. A single delete message can contain multiple delete operations. -[source,xml] +[tabs#xml-delete-request] +====== +V1 API:: ++ +==== +[source,bash] ---- +curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary ' 0002166313 0031745983 subject:sport publisher:penguin - +' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/xml' -H "Content-Type: text/xml" --data-binary ' + + 0002166313 + 0031745983 + subject:sport + publisher:penguin +' +---- +==== +====== [IMPORTANT] ==== @@ -344,12 +402,41 @@ The rollback command rolls back all add and deletes made to the index since the It neither calls any event listeners nor creates a new searcher. Its syntax is simple: ``. +The same command can be sent through either API: + +[tabs#xml-rollback-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary '\' +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/xml' -H "Content-Type: text/xml" --data-binary '\' +---- +==== +====== + ==== Grouping Operations You can post several commands in a single XML file by grouping them with the surrounding `` element. -[source,xml] +[tabs#xml-grouped-update-request] +====== +V1 API:: ++ +==== +[source,bash] ---- +curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary ' @@ -360,62 +447,35 @@ You can post several commands in a single XML file by grouping them with the sur 0002166313 - +' ---- +==== - -=== Using curl to Perform Updates - -You can use the `curl` utility to perform any of the above commands, using its `--data-binary` option to append the XML message to the `curl` command, and generating a HTTP POST request. -For example: - -[source,bash] ----- -curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary ' - - - Patrick Eagar - Sports - 796.35 - 0002166313 - 1982 - Collins - -' ----- - -For posting XML messages contained in a file, you can use the alternative form: - -[source,bash] ----- -curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary @myfile.xml ----- - -The approach above works well, but using the `--data-binary` option causes `curl` to load the whole `myfile.xml` into memory before posting it to server. -This may be problematic when dealing with multi-gigabyte files. -This alternative `curl` command performs equivalent operations but with minimal `curl` memory usage: - +V2 API:: ++ +==== [source,bash] ---- -curl -X POST 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" -T "myfile.xml" ----- - -Responses from Solr take the form shown here: - -[source,xml] ----- - - - 0 - 127 - - +curl 'http://localhost:8983/api/collections/{collectionName}/update/xml' -H "Content-Type: text/xml" --data-binary ' + + + + + + + + + 0002166313 + +' ---- - -The status field will be non-zero in case of failure. +==== +====== === Using XSLT to Transform XML Index Updates +NOTE: The XSLT update handler is V1-only. V2 does not provide a configurable equivalent for custom request handlers such as `/update/xslt`; use the V1 endpoint for these requests. + The xref:configuration-guide:script-update-processor.adoc[Scripting module] provides a separate XSLT Update Request Handler that allows you to index any arbitrary XML by using the `` parameter to apply an https://en.wikipedia.org/wiki/XSLT[XSL transformation]. You must have an XSLT stylesheet in the `conf/xslt` directory of your xref:configuration-guide:config-sets.adoc[configset] that can transform the incoming data to the expected `` format, and use the `tr` parameter to specify the name of that stylesheet. @@ -494,9 +554,9 @@ One example usage would be to copy a Solr 1.3 index (which does not have CSV res [source,bash] ---- -curl -o standard_solr_xml_format.xml 'http://localhost:8983/solr/techproducts/select?q=ipod&fl=id,cat,name,popularity,price,score&wt=xml' +curl 'http://localhost:8983/solr/techproducts/select?q=ipod&fl=id,cat,name,popularity,price,score&wt=xml' -o standard_solr_xml_format.xml -curl 'http://localhost:8983/solr/techproducts/update/xslt?commit=true&tr=updateXml.xsl' -H "Content-Type: text/xml" -d @standard_solr_xml_format.xml +curl 'http://localhost:8983/solr/techproducts/update/xslt?commit=true&tr=updateXml.xsl' -H 'Content-Type: text/xml' --data-binary @standard_solr_xml_format.xml ---- NOTE: You can see the opposite export/import cycle using the `tr` parameter in the xref:query-guide:response-writers.adoc#xslt-writer-example[Response Writer XSLT example]. @@ -507,10 +567,26 @@ CSV formatted update requests may be sent to Solr's `/update` handler using `Con A sample CSV file is provided at `example/exampledocs/books.csv` that you can use to add some documents to the Solr "techproducts" example: +[tabs#csv-file-request] +====== +V1 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/{collectionName}/update?commit=true' --data-binary @example/exampledocs/books.csv -H 'Content-type:application/csv' +curl 'http://localhost:8983/solr/{collectionName}/update?commit=true' -H 'Content-type:application/csv' --data-binary @example/exampledocs/books.csv ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update?commit=true' -H 'Content-type:application/csv' --data-binary @example/exampledocs/books.csv +---- +==== +====== === CSV Update Parameters @@ -662,17 +738,7 @@ Example: `map=left:right` or `f.subject.map=history:bunk` If `true`, split a field into multiple values by a separate parser. This parameter is used on a per-field basis, for example `f.FIELD_NAME_GOES_HERE.split=true`. -`overwrite`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: `true` -|=== -+ -If `true`, check for and overwrite duplicate documents, based on the uniqueKey field declared in the Solr schema. -If you know the documents you are indexing do not contain any duplicates then you may see a considerable speed up setting this to `false`. -+ -This parameter is global. +`overwrite` is a common update option; see <> for its behavior. `commit`:: + @@ -684,17 +750,7 @@ This parameter is global. Issues a commit after the data has been ingested. This parameter is global. -`commitWithin`:: -+ -[%autowidth,frame=none] -|=== -|Optional |Default: none -|=== -+ -Add the document within the specified number of milliseconds. -This parameter is global. -+ -Example: `commitWithin=10000` +`commitWithin` is a common update option; see <> for its behavior. `rowid`:: + @@ -733,19 +789,245 @@ SELECT * INTO OUTFILE '/tmp/result.txt' FROM mytable; This file could then be imported into Solr by setting the `separator` to tab (%09) and the `escape` to backslash (%5c). +[tabs#csv-tab-delimited-request] +====== +V1 API:: ++ +==== [source,bash] ---- curl 'http://localhost:8983/solr/{collectionName}/update/csv?commit=true&separator=%09&escape=%5c' --data-binary @/tmp/result.txt ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/csv?commit=true&separator=%09&escape=%5c' --data-binary @/tmp/result.txt +---- +==== +====== === CSV Update Convenience Paths -In addition to the `/update` handler, there is an additional CSV specific request handler path available by default in Solr, that implicitly override the behavior of some request parameters: +The `/update/csv` endpoint is a specialized Solr request path designed for importing CSV data. +You use it primarily when your client application or script cannot easily set the required `Content-Type: text/csv` header, as this route automatically forces Solr to treat all incoming payloads as CSV data without needing extra parameter configuration. + +== Commit and Optimize + +The `` operation writes all documents loaded since the last commit to one or more segment files on the disk. +Before a commit has been issued, newly indexed content is not visible to searches. +The commit operation opens a new searcher, and triggers any event listeners that have been configured. + +Commits may be issued explicitly with a `` message, and can also be triggered from `` parameters in `solrconfig.xml`. + +The `` operation requests Solr to merge internal data structures. +For a large index, optimization will take some time to complete, but by merging many small segment files into larger segments, search performance may improve. +If you are using Solr's replication mechanism to distribute searches across many systems, be aware that after an optimize, a complete index will need to be transferred. + +WARNING: You should only consider using optimize on static indexes, i.e., indexes that can be optimized as part of the regular update process (say once-a-day updates). +Applications requiring NRT functionality should not use optimize. + +The `` and `` elements accept these optional attributes: -[cols=",",options="header",] +`waitSearcher`:: ++ +[%autowidth,frame=none] |=== -|Path |Default Parameters -|`/update/csv` |`update.contentType=application/csv` +|Optional |Default: `true` |=== ++ +Blocks until a new searcher is opened and registered as the main query searcher, making the changes visible. -The `/update/csv` path may be useful for clients sending in CSV formatted update commands from applications where setting the Content-Type proves difficult. +`expungeDeletes`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: `false` +|=== ++ +Merges segments that have more than 10% deleted docs, expunging the deleted documents in the process. +Resulting segments will respect `maxMergedSegmentMB`. +This option only applies in a `` operation. ++ +WARNING: `expungeDeletes` is less expensive than optimize, but the same warnings apply. + +`maxSegments`:: ++ +[%autowidth,frame=none] +|=== +|Optional |Default: none +|=== ++ +Makes a best effort attempt to merge the segments down to no more than this number of segments but does not guarantee that the goal will be achieved. +Unless there is tangible evidence that optimizing to a small number of segments is beneficial, this parameter should be omitted and the default behavior accepted. +This option only applies in a `` operation. +Default is unlimited, resulting segments respect the `maxMergedSegmentMB` setting. + +== Common Update Options + +These options apply to document update requests in JSON, XML, and CSV formats. +They can be supplied as URL parameters; formats that support command bodies may also provide them in the body using their format-specific syntax. + +`commitWithin`:: ++ +Requests that Solr commit the update within the specified number of milliseconds. + +`overwrite`:: ++ +Controls whether Solr checks the unique key and replaces an existing document with the same key. +The default is `true`. +Set it to `false` when the input is known not to contain duplicate unique keys and replacing existing documents is unnecessary. + +== Versioned Update Responses + +Add the `versions=true` query parameter to an update request when the response must include the versions assigned to add or delete operations. +This parameter is supported by both the V1 update handlers and the V2 update endpoints. +It can be used with JSON, XML, CSV, Javabin, and CBOR update requests where the format is supported by the endpoint. +The `adds` field applies to document-ingest formats; `deletes` and `deleteByQuery` apply to formats that accept delete commands, such as JSON command updates, XML, and Javabin. + +See xref:partial-document-updates.adoc#optimistic-concurrency[Optimistic Concurrency] for more information about versioned updates. + +[tabs#versioned-update-response] +====== +V1 API:: ++ +==== +V1 returns alternating name/version values. +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 127 + }, + "adds": [ + "0002166313", + 123456789 + ], + "deletes": [ + "0002166314", + -123456790 + ], + "deleteByQuery": [ + "status:deleted", + -123456791 + ] +} +---- +==== + +V2 API:: ++ +==== +The v2 API returns typed entries: + +[source,json] +---- +{ + "responseHeader": { + "status": 0, + "QTime": 127 + }, + "adds": [ + { + "id": "0002166313", + "version": 123456789 + } + ], + "deletes": [ + { + "id": "0002166314", + "version": -123456790 + } + ], + "deleteByQuery": [ + { + "query": "status:deleted", + "version": -123456791 + } + ] +} +---- +==== +====== + +The status field will be non-zero in case of failure. + +The response fields represent: + +* `adds`: Documents added and the versions assigned to them. +* `deletes`: Documents deleted and the versions assigned to their delete operations. +* `deleteByQuery`: Delete-by-query operations and the versions assigned to them. + +The version value is assigned by Solr. Delete versions are commonly negative. + +== Javabin Document Indexing + +Javabin is Solr's native binary update format and is primarily intended for SolrJ clients. +V1 clients send Javabin to `/update` with `Content-Type: application/javabin`. +V2 clients can use `/update/javabin` or the general `/update` endpoint with the same content type. + +== UpdateRequestHandler Configuration + +The default configuration file has the update request handler configured by default. + +The recommended way to configure and use request handlers is with path-based names that map directly to paths in the request URL. +You can register a handler at more than one path when you need different defaults for different uses. + +[source,xml] +---- + +---- + +== Posting Large Update Files + +Large update files can be posted through either API. The examples below use XML; use the corresponding content type and endpoint for CSV, Javabin, or another supported format. +Using `-d @myfile.xml` causes `curl` to load the whole file into memory before posting it to the server. +This may be problematic for multi-gigabyte files. + +[tabs#large-file-update-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" --data-binary @myfile.xml +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/xml' -H "Content-Type: text/xml" --data-binary @myfile.xml +---- +==== +====== + +For minimal `curl` memory usage, stream the file with `-T`: + +[tabs#large-file-streaming-request] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/{collectionName}/update' -H "Content-Type: text/xml" -T "myfile.xml" -X POST +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/{collectionName}/update/xml' -H "Content-Type: text/xml" -T "myfile.xml" -X POST +---- +==== +====== diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/partial-document-updates.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/partial-document-updates.adoc index 5c6f913251e8..2e59fb0e92b9 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/partial-document-updates.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/partial-document-updates.adoc @@ -64,6 +64,9 @@ May be specified as a single value, or as a list. Increments or decrements a numeric field's value by a specific amount, specified as a single integer or float. Positive amounts increment the field's value, and negative decrement. +NOTE: Atomic update syntax is supported exclusively on the `/update` endpoint. +Document-only JSON paths (such as `/update/json/docs` in V1 or `/update/json` in V2) index documents literally; if a field value matches a modifier format, these endpoints will reject the document instead of applying the atomic update. + === Field Storage The core functionality of atomically updating a document requires that all fields in your schema must be configured as stored (`stored="true"`) or docValues (`docValues="true"`) except for fields which are `` destinations, which must be configured as `stored="false"` and either `docValues="false"` or `useDocValuesAsStored="false"`. @@ -85,40 +88,63 @@ If the following document exists in our collection: [source,json] ---- -{"id":"mydoc", - "price":10, - "popularity":42, - "categories":["kids"], - "sub_categories":["under_5","under_10"], - "promo_ids":["a123x"], - "tags":["free_to_try","buy_now","clearance","on_sale"] +{"id":"SOLR1000", + "name":"Solr, the Enterprise Search Server", + "manu":"Apache Software Foundation", + "cat":["software","search"], + "features":["Advanced Full-Text Search Capabilities using Lucene", + "Optimized for High Volume Web Traffic"], + "price":0.0, + "popularity":10 } ---- And we apply the following update command: -[source,json] +[tabs#atomic-update-request] +====== +V1 API:: ++ +==== +[source,bash] ---- -{"id":"mydoc", - "price":{"set":99}, - "popularity":{"inc":-7}, - "categories":{"add":["toys","games"]}, - "sub_categories":{"add-distinct":"under_10"}, - "promo_ids":{"remove":"a123x"}, - "tags":{"remove":["free_to_try","on_sale"]} -} +curl 'http://localhost:8983/solr/techproducts/update' -H 'Content-Type: application/json' --data-binary '[ + {"id":"SOLR1000", + "price":{"set":9.99}, + "popularity":{"inc":5}, + "cat":{"add":"enterprise"}, + "features":{"remove":"Optimized for High Volume Web Traffic"}} +]' +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update' -H 'Content-Type: application/json' --data-binary '[ + {"id":"SOLR1000", + "price":{"set":9.99}, + "popularity":{"inc":5}, + "cat":{"add":"enterprise"}, + "features":{"remove":"Optimized for High Volume Web Traffic"}} +]' ---- +==== +====== The resulting document in our collection will be: [source,json] ---- -{"id":"mydoc", - "price":99, - "popularity":35, - "categories":["kids","toys","games"], - "sub_categories":["under_5","under_10"], - "tags":["buy_now","clearance"] +{"id":"SOLR1000", + "name":"Solr, the Enterprise Search Server", + "manu":"Apache Software Foundation", + "cat":["software","search","enterprise"], + "features":["Advanced Full-Text Search Capabilities using Lucene"], + "price":9.99, + "popularity":15 } ---- @@ -155,79 +181,281 @@ This is how Solr understands that you are updating a child document, and not a R All of the examples below use `id` prefixes, so no `\_route_` parameter will be necessary for these examples. ==== -For the upcoming examples, we'll assume an index containing the same documents covered in xref:indexing-nested-documents.adoc#example-indexing-syntax[Indexing Nested Documents]: +For the upcoming examples, we'll use a nested MacBook Pro product block. +The block must be indexed before applying the child updates below. -include::indexing-nested-documents.adoc[tag=sample-indexing-deeply-nested-documents] +The following request creates the sample block used in the first child-document example: + +[tabs#child-document-sample] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ + { + "id": "APPLE-MACBOOK-PRO-14", + "type_s": "product", + "name": "MacBook Pro 14-inch", + "manu": "Apple", + "cat": ["computers", "laptops"], + "features": ["14-inch Liquid Retina XDR display", "Thunderbolt 4", "MagSafe charging"], + "skus": [ + { + "id": "APPLE-MBP14-M3-16-512", + "type_s": "sku", + "processor_s": "Apple M3 Pro", + "ram_gb_i": 16, + "storage_gb_i": 512, + "color_s": "Space Black", + "price": 1999.00, + "inventory_i": 24, + "manuals": [ + { + "id": "APPLE-MBP14-MANUAL-EN", + "type_s": "manual", + "language_s": "en", + "title_t": "MacBook Pro User Guide" + } + ] + }, + { + "id": "APPLE-MBP14-M3-36-1TB", + "type_s": "sku", + "processor_s": "Apple M3 Pro", + "ram_gb_i": 36, + "storage_gb_i": 1024, + "color_s": "Silver", + "price": 2399.00, + "inventory_i": 11 + } + ] + } +]' +---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ + { + "id": "APPLE-MACBOOK-PRO-14", + "type_s": "product", + "name": "MacBook Pro 14-inch", + "manu": "Apple", + "cat": ["computers", "laptops"], + "features": ["14-inch Liquid Retina XDR display", "Thunderbolt 4", "MagSafe charging"], + "skus": [ + { + "id": "APPLE-MBP14-M3-16-512", + "type_s": "sku", + "processor_s": "Apple M3 Pro", + "ram_gb_i": 16, + "storage_gb_i": 512, + "color_s": "Space Black", + "price": 1999.00, + "inventory_i": 24, + "manuals": [ + { + "id": "APPLE-MBP14-MANUAL-EN", + "type_s": "manual", + "language_s": "en", + "title_t": "MacBook Pro User Guide" + } + ] + }, + { + "id": "APPLE-MBP14-M3-36-1TB", + "type_s": "sku", + "processor_s": "Apple M3 Pro", + "ram_gb_i": 36, + "storage_gb_i": 1024, + "color_s": "Silver", + "price": 2399.00, + "inventory_i": 11 + } + ] + } +]' +---- +==== +====== ==== Modifying Child Document Fields All of the <> mentioned above are supported for "real" fields of Child Documents: +[tabs#child-field-update] +====== +V1 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/gettingstarted/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ { - "id": "P11!S31", - "_root_": "P11!prod", - "price_i": { "inc": 73 }, - "color_s": { "set": "GREY" } + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "price": { "inc": 100 }, + "inventory_i": { "inc": -1 } } ]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +{ + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "price": { "inc": 100 }, + "inventory_i": { "inc": -1 } +} ]' +---- +==== +====== ==== Replacing All Child Documents As with normal (multiValued) fields, the `set` keyword can be used to replace all child documents in a pseudo-field: +[tabs#child-replace-update] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +{ + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "manuals": { "set": [ { "id": "APPLE-MBP14-MANUAL-EN", + "type_s": "manual", + "language_s": "en", + "title_t": "MacBook Pro User Guide" + }, + { "id": "APPLE-MBP14-MANUAL-FR", + "type_s": "manual", + "language_s": "fr", + "title_t": "Guide de l’utilisateur du MacBook Pro" + } ] } + +} ]' +---- +==== + +V2 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/gettingstarted/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ { - "id": "P22!S22", - "_root_": "P22!prod", - "manuals": { "set": [ { "id": "P22!D77", - "name_s": "Why Red Pens Are the Best", - "content_t": "... correcting papers ...", + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "manuals": { "set": [ { "id": "APPLE-MBP14-MANUAL-EN", + "type_s": "manual", + "language_s": "en", + "title_t": "MacBook Pro User Guide" }, - { "id": "P22!D88", - "name_s": "How to get Red ink stains out of fabric", - "content_t": "... vinegar ...", + { "id": "APPLE-MBP14-MANUAL-FR", + "type_s": "manual", + "language_s": "fr", + "title_t": "Guide de l’utilisateur du MacBook Pro" } ] } } ]' ---- +==== +====== ==== Adding a Child Document As with normal (multiValued) fields, the `add` keyword can be used to add additional child documents to a pseudo-field: +[tabs#child-add-update] +====== +V1 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +{ + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "manuals": { "add": { "id": "APPLE-MBP14-MANUAL-ES", + "type_s": "manual", + "language_s": "es", + "title_t": "Guía del usuario del MacBook Pro" + } } +} ]' +---- +==== + +V2 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/gettingstarted/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ { - "id": "P11!S21", - "_root_": "P11!prod", - "manuals": { "add": { "id": "P11!D99", - "name_s": "Why Red Staplers Are the Best", - "content_t": "Once upon a time, Mike Judge ...", + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "manuals": { "add": { "id": "APPLE-MBP14-MANUAL-ES", + "type_s": "manual", + "language_s": "es", + "title_t": "Guía del usuario del MacBook Pro" } } } ]' ---- +==== +====== -Note that this is add-or-replace (by ID). Meaning, if it happens that doc `P11!S21` already has a child doc with the ID `P11!D99` (the one we are adding), then it will be replaced. +Note that this is add-or-replace (by ID). Meaning, if it happens that doc `APPLE-MBP14-M3-16-512` already has a child doc with the ID `APPLE-MBP14-MANUAL-ES` (the one we are adding), then it will be replaced. ==== Removing a Child Document As with normal (multiValued) fields, the `remove` keyword can be used to remove a child document (by `id`) from it's pseudo-field: +[tabs#child-remove-update] +====== +V1 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/gettingstarted/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ { - "id": "P11!S21", - "_root_": "P11!prod", - "manuals": { "remove": { "id": "P11!D41" } } + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "manuals": { "remove": { "id": "APPLE-MBP14-MANUAL-ES" } } } ]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[ +{ + "id": "APPLE-MBP14-M3-16-512", + "_root_": "APPLE-MACBOOK-PRO-14", + "manuals": { "remove": { "id": "APPLE-MBP14-MANUAL-ES" } } +} ]' +---- +==== +====== == In-Place Updates @@ -266,11 +494,11 @@ Users can specify this option when they would prefer that an update request "fai === In-Place Update Example -If the price and popularity fields are defined in the schema as: +If your schema defines the price and popularity fields like the techproducts example: -`` +`` -`` +`` [TIP] ==== @@ -280,42 +508,54 @@ If the price and popularity fields are defined in the schema as: `docValues="true"` is the default for schemas with version >= `1.7`, so it can be omitted. ==== -If the following document exists in our collection: +Assuming the following document exists in our collection: [source,json] ---- { - "id":"mydoc", - "price":10, - "popularity":42, - "categories":["kids"], - "promo_ids":["a123x"], - "tags":["free_to_try","buy_now","clearance","on_sale"] + "id":"SOLR1000", + "name":"Solr, the Enterprise Search Server", + "manu":"Apache Software Foundation", + "cat":["software","search"], + "price":0.0, + "popularity":10 } ---- And we apply the following update command: -[source,json] +[tabs#in-place-update-request] +====== +V1 API:: ++ +==== +[source,bash] ---- -{ - "id":"mydoc", - "price":{"set":99}, - "popularity":{"inc":20} -} +curl 'http://localhost:8983/solr/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[{"id":"SOLR1000","price":{"set":1.0},"popularity":{"inc":1}}]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?commit=true' -H 'Content-Type: application/json' --data-binary '[{"id":"SOLR1000","price":{"set":1.0},"popularity":{"inc":1}}]' +---- +==== +====== The resulting document in our collection will be: [source,json] ---- { - "id":"mydoc", - "price":99, - "popularity":62, - "categories":["kids"], - "promo_ids":["a123x"], - "tags":["free_to_try","buy_now","clearance","on_sale"] + "id":"SOLR1000", + "name":"Solr, the Enterprise Search Server", + "manu":"Apache Software Foundation", + "cat":["software","search"], + "price":1.0, + "popularity":11 } ---- @@ -354,29 +594,84 @@ This allows clients to immediately know what the `\_version_` is of every docume Following are some examples using `versions=true` in queries: +[tabs#optimistic-add] +====== +V1 API:: ++ +==== [source,bash] ---- curl 'http://localhost:8983/solr/techproducts/update?versions=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' -[ { "id" : "aaa" }, - { "id" : "bbb" } ]' +[ { "id" : "SOLR1000" }, + { "id" : "0579B002" } ]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?versions=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[ { "id" : "SOLR1000" }, + { "id" : "0579B002" } ]' +---- +==== +====== +[tabs#optimistic-add-response] +====== +V1 API:: ++ +==== +[source,json] +---- +{ + "adds":[ + "SOLR1000",1632740120218042368, + "0579B002",1632740120250548224]} +---- +==== + +V2 API:: ++ +==== [source,json] ---- { "adds":[ - "aaa",1632740120218042368, - "bbb",1632740120250548224]} + {"id":"SOLR1000","version":1632740120218042368}, + {"id":"0579B002","version":1632740120250548224}]} ---- +==== +====== -In this example, we have added 2 documents "aaa" and "bbb". +In this example, we have added 2 documents "SOLR1000" and "0579B002". Because we added `versions=true` to the request, the response shows the document version for each document. +[tabs#optimistic-wrong-version] +====== +V1 API:: ++ +==== [source,bash] ---- curl 'http://localhost:8983/solr/techproducts/update?_version_=999999&versions=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' - [{ "id" : "aaa", - "foo_s" : "update attempt with wrong existing version" }]' + [{ "id" : "SOLR1000", + "name" : "update attempt with wrong existing version" }]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?_version_=999999&versions=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' + [{ "id" : "SOLR1000", + "name" : "update attempt with wrong existing version" }]' +---- +==== +====== [source,json] ---- { @@ -384,35 +679,89 @@ curl 'http://localhost:8983/solr/techproducts/update?_version_=999999&versions=t "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","org.apache.solr.common.SolrException"], - "msg":"version conflict for aaa expected=999999 actual=1632740120218042368", + "msg":"version conflict for SOLR1000 expected=999999 actual=1632740120218042368", "code":409}} ---- -In this example, we've attempted to update document "aaa" but specified the wrong version in the request: `_version_=999999` doesn't match the document version we just got when we added the document. +In this example, we've attempted to update document "SOLR1000" but specified the wrong version in the request: `_version_=999999` doesn't match the document version we just got when we added the document. We get an error in response. +[tabs#optimistic-correct-version] +====== +V1 API:: ++ +==== [source,bash] ---- curl 'http://localhost:8983/solr/techproducts/update?_version_=1632740120218042368&versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' -[{ "id" : "aaa", - "foo_s" : "update attempt with correct existing version" }]' +[{ "id" : "SOLR1000", + "name" : "update attempt with correct existing version" }]' +---- +==== + +V2 API:: ++ +==== +[source,bash] ---- +curl 'http://localhost:8983/api/collections/techproducts/update?_version_=1632740120218042368&versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[{ "id" : "SOLR1000", + "name" : "update attempt with correct existing version" }]' +---- +==== +====== +[tabs#optimistic-correct-response] +====== +V1 API:: ++ +==== [source,json] ---- { "adds":[ - "aaa",1632740462042284032]} + "SOLR1000",1632740462042284032]} ---- +==== + +V2 API:: ++ +==== +[source,json] +---- +{ + "adds":[ + {"id":"SOLR1000","version":1632740462042284032}]} +---- +==== +====== Now we've sent an update with a value for `\_version_` that matches the value in the index, and it succeeds. Because we included `versions=true` to the update request, the response includes a different value for the `\_version_` field. +[tabs#optimistic-embedded-wrong-version] +====== +V1 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update?&versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' -[{ "id" : "aaa", _version_ : 100, - "foo_s" : "update attempt with wrong existing version embedded in document" }]' +curl 'http://localhost:8983/solr/techproducts/update?versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[{ "id" : "SOLR1000", "_version_" : 100, + "name" : "update attempt with wrong existing version embedded in document" }]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[{ "id" : "SOLR1000", "_version_" : 100, + "name" : "update attempt with wrong existing version embedded in document" }]' +---- +==== +====== [source,json] ---- { @@ -420,7 +769,7 @@ curl 'http://localhost:8983/solr/techproducts/update?&versions=true&commit=true& "metadata":[ "error-class","org.apache.solr.common.SolrException", "root-error-class","org.apache.solr.common.SolrException"], - "msg":"version conflict for aaa expected=100 actual=1632740462042284032", + "msg":"version conflict for SOLR1000 expected=100 actual=1632740462042284032", "code":409}} ---- @@ -428,18 +777,54 @@ Now we've sent an update with a value for `\_version_` embedded in the document This request fails because we have specified the wrong version. This is useful when documents are sent in a batch and different `\_version_` values need to be specified for each doc. +[tabs#optimistic-embedded-correct-version] +====== +V1 API:: ++ +==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update?&versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' -[{ "id" : "aaa", _version_ : 1632740462042284032, - "foo_s" : "update attempt with correct version embedded in document" }]' +curl 'http://localhost:8983/solr/techproducts/update?versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[{ "id" : "SOLR1000", "_version_" : 1632740462042284032, + "name" : "update attempt with correct version embedded in document" }]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?versions=true&commit=true&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[{ "id" : "SOLR1000", "_version_" : 1632740462042284032, + "name" : "update attempt with correct version embedded in document" }]' +---- +==== +====== +[tabs#optimistic-embedded-correct-response] +====== +V1 API:: ++ +==== +[source,json] +---- +{ + "adds":[ + "SOLR1000",1632741942747987968]} +---- +==== + +V2 API:: ++ +==== [source,json] ---- { "adds":[ - "aaa",1632741942747987968]} + {"id":"SOLR1000","version":1632741942747987968}]} ---- +==== +====== Now we've sent an update with a value for `\_version_` embedded in the document itself. This request fails because we have specified the wrong version. @@ -456,32 +841,68 @@ curl 'http://localhost:8983/solr/techproducts/query?q=*:*&fl=id,_version_&omitHe { "response":{"numFound":3,"start":0,"docs":[ { "_version_":1632740120250548224, - "id":"bbb"}, + "id":"0579B002"}, { "_version_":1632741942747987968, - "id":"aaa"}] + "id":"SOLR1000"}] }} ---- Finally, we can issue a query that requests the `\_version_` field be included in the response, and we can see that for the two documents in our example index. +[tabs#optimistic-fail-conflicts] +====== +V1 API:: ++ +==== [source,bash] ---- curl 'http://localhost:8983/solr/techproducts/update?versions=true&_version_=-1&failOnVersionConflicts=false&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' -[ { "id" : "aaa" }, - { "id" : "ccc" } ]' +[ { "id" : "SOLR1000" }, + { "id" : "IW-02" } ]' ---- +==== + +V2 API:: ++ +==== +[source,bash] +---- +curl 'http://localhost:8983/api/collections/techproducts/update?versions=true&_version_=-1&failOnVersionConflicts=false&omitHeader=true' -H 'Content-Type: application/json' --data-binary ' +[ { "id" : "SOLR1000" }, + { "id" : "IW-02" } ]' +---- +==== +====== +[tabs#optimistic-fail-response] +====== +V1 API:: ++ +==== +[source,json] +---- +{ + "adds":[ + "IW-02",1632740949182382080]} +---- +==== + +V2 API:: ++ +==== [source,json] ---- { "adds":[ - "ccc",1632740949182382080]} + {"id":"IW-02","version":1632740949182382080}]} ---- +==== +====== -In this example, we have added 2 documents "aaa" and "ccc". -As we have specified the parameter `\_version_=-1`, this request should not add the document with the id `aaa` because it already exists. +In this example, we have added 2 documents "SOLR1000" and "IW-02". +As we have specified the parameter `\_version_=-1`, this request should not add the document with the id `SOLR1000` because it already exists. The request succeeds & does not throw any error because the `failOnVersionConflicts=false` parameter is specified. -The response shows that only document `ccc` is added and `aaa` is silently ignored. +The response shows that only document `IW-02` is added and `SOLR1000` is silently ignored. For more information, please also see Yonik Seeley's presentation on https://www.youtube.com/watch?v=WYVM6Wz-XTw[NoSQL features in Solr 4] from Apache Lucene EuroCon 2012. diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/transforming-and-indexing-custom-json.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/transforming-and-indexing-custom-json.adoc index 2e2bd80c217b..c4ea56dba1d8 100644 --- a/solr/solr-ref-guide/modules/indexing-guide/pages/transforming-and-indexing-custom-json.adoc +++ b/solr/solr-ref-guide/modules/indexing-guide/pages/transforming-and-indexing-custom-json.adoc @@ -20,7 +20,7 @@ If you have JSON documents that you would like to index without transforming them into Solr's structure, you can add them to Solr by including some parameters with the update request. These parameters provide information on how to split a single JSON file into multiple Solr documents and how to map fields to Solr's schema. -One or more valid JSON documents can be sent to the `/update/json/docs` path with the configuration params. +In V1, one or more valid JSON documents can be sent to `/update/json/docs` with the configuration parameters. In V2, use the document-only `/update/json` endpoint. == Mapping Parameters @@ -115,7 +115,8 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ '&f=subject:/exams/subject'\ '&f=test:/exams/test'\ '&f=marks:/exams/marks'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -134,44 +135,12 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ ---- ==== -V2 API User-Managed / Standalone Solr:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/cores/techproducts/update/json/docs'\ -'?split=/exams'\ -'&f=first:/first'\ -'&f=last:/last'\ -'&f=grade:/grade'\ -'&f=subject:/exams/subject'\ -'&f=test:/exams/test'\ -'&f=marks:/exams/marks'\ - -H 'Content-type:application/json' -d ' -{ - "first": "John", - "last": "Doe", - "grade": 8, - "exams": [ - { - "subject": "Maths", - "test" : "term1", - "marks" : 90}, - { - "subject": "Biology", - "test" : "term1", - "marks" : 86} - ] -}' ----- -==== - -V2 API SolrCloud:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/collections/techproducts/update/json/docs'\ +curl 'http://localhost:8983/api/collections/techproducts/update/json'\ '?split=/exams'\ '&f=first:/first'\ '&f=last:/last'\ @@ -179,7 +148,8 @@ curl 'http://localhost:8983/api/collections/techproducts/update/json/docs'\ '&f=subject:/exams/subject'\ '&f=test:/exams/test'\ '&f=marks:/exams/marks'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -242,7 +212,8 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ '&f=/exams/subject'\ '&f=/exams/test'\ '&f=/exams/marks'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -261,44 +232,12 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ ---- ==== -V2 API User-Managed / Standalone Solr:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/cores/techproducts/update/json/docs'\ -'?split=/exams'\ -'&f=/first'\ -'&f=/last'\ -'&f=/grade'\ -'&f=/exams/subject'\ -'&f=/exams/test'\ -'&f=/exams/marks'\ - -H 'Content-type:application/json' -d ' -{ - "first": "John", - "last": "Doe", - "grade": 8, - "exams": [ - { - "subject": "Maths", - "test" : "term1", - "marks" : 90}, - { - "subject": "Biology", - "test" : "term1", - "marks" : 86} - ] -}' ----- -==== - -V2 API SolrCloud:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/collections/techproducts/update/json/docs'\ +curl 'http://localhost:8983/api/collections/techproducts/update/json'\ '?split=/exams'\ '&f=/first'\ '&f=/last'\ @@ -306,7 +245,8 @@ curl 'http://localhost:8983/api/collections/techproducts/update/json/docs'\ '&f=/exams/subject'\ '&f=/exams/test'\ '&f=/exams/marks'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -350,7 +290,7 @@ V1 API:: ==== [source,bash] ---- - curl 'http://localhost:8983/solr/techproducts/config/params' -H 'Content-type:application/json' -d '{ +curl 'http://localhost:8983/solr/techproducts/config/params' -H 'Content-Type: application/json' -d '{ "set": { "my_params": { "split": "/exams", @@ -359,26 +299,12 @@ V1 API:: ---- ==== -V2 API User-Managed / Standalone Solr:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/cores/techproducts/config/params' -H 'Content-type:application/json' -d '{ - "set": { - "my_params": { - "split": "/exams", - "f": ["first:/first","last:/last","grade:/grade","subject:/exams/subject","test:/exams/test"] - }}}' ----- -==== - -V2 API SolrCloud:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/collections/techproducts/config/params' -H 'Content-type:application/json' -d '{ +curl 'http://localhost:8983/api/collections/techproducts/config/params' -H 'Content-Type: application/json' -d '{ "set": { "my_params": { "split": "/exams", @@ -397,31 +323,10 @@ V1 API:: ==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update/json/docs?useParams=my_params' -H 'Content-type:application/json' -d '{ - "first": "John", - "last": "Doe", - "grade": 8, - "exams": [{ - "subject": "Maths", - "test": "term1", - "marks": 90 - }, - { - "subject": "Biology", - "test": "term1", - "marks": 86 - } - ] -}' ----- -==== - -V2 API User-Managed / Standalone Solr:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/cores/techproducts/update/json?useParams=my_params' -H 'Content-type:application/json' -d '{ +curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ +'?useParams=my_params'\ + -H 'Content-Type: application/json' \ + -d '{ "first": "John", "last": "Doe", "grade": 8, @@ -440,12 +345,15 @@ curl 'http://localhost:8983/api/cores/techproducts/update/json?useParams=my_para ---- ==== -V2 API SolrCloud:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/collections/techproducts/update/json?useParams=my_params' -H 'Content-type:application/json' -d '{ +curl 'http://localhost:8983/api/collections/techproducts/update/json'\ +'?useParams=my_params'\ + -H 'Content-Type: application/json' \ + -d '{ "first": "John", "last": "Doe", "grade": 8, @@ -494,7 +402,8 @@ V1 API:: curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ '?split=/exams'\ '&f=/**'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -513,34 +422,7 @@ curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ ---- ==== -V2 API User-Managed / Standalone Solr:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/cores/techproducts/update/json'\ -'?split=/exams'\ -'&f=/**'\ - -H 'Content-type:application/json' -d ' -{ - "first": "John", - "last": "Doe", - "grade": 8, - "exams": [ - { - "subject": "Maths", - "test" : "term1", - "marks" : 90}, - { - "subject": "Biology", - "test" : "term1", - "marks" : 86} - ] -}' ----- -==== - -V2 API SolrCloud:: +V2 API:: + ==== [source,bash] @@ -548,7 +430,8 @@ V2 API SolrCloud:: curl 'http://localhost:8983/api/collections/techproducts/update/json'\ '?split=/exams'\ '&f=/**'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -583,34 +466,8 @@ V1 API:: curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ '?split=/'\ '&f=txt:/**'\ - -H 'Content-type:application/json' -d ' -{ - "first": "John", - "last": "Doe", - "grade": 8, - "exams": [ - { - "subject": "Maths", - "test" : "term1", - "marks" : 90}, - { - "subject": "Biology", - "test" : "term1", - "marks" : 86} - ] -}' ----- -==== - -V2 API User-Managed / Standalone Solr:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/cores/techproducts/update/json'\ -'?split=/'\ -'&f=txt:/**'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -629,7 +486,7 @@ curl 'http://localhost:8983/api/cores/techproducts/update/json'\ ---- ==== -V2 API SolrCloud:: +V2 API:: + ==== [source,bash] @@ -637,7 +494,8 @@ V2 API SolrCloud:: curl 'http://localhost:8983/api/collections/techproducts/update/json'\ '?split=/'\ '&f=txt:/**'\ - -H 'Content-type:application/json' -d ' + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -670,33 +528,10 @@ V1 API:: ==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update/json/docs?split=/exams'\ - -H 'Content-type:application/json' -d ' -{ - "first": "John", - "last": "Doe", - "grade": 8, - "exams": [ - { - "subject": "Maths", - "test" : "term1", - "marks" : 90}, - { - "subject": "Biology", - "test" : "term1", - "marks" : 86} - ] -}' ----- -==== - -V2 API User-Managed / Standalone Solr:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/cores/techproducts/update/json?split=/exams'\ - -H 'Content-type:application/json' -d ' +curl 'http://localhost:8983/solr/techproducts/update/json/docs'\ +'?split=/exams'\ + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -715,13 +550,15 @@ curl 'http://localhost:8983/api/cores/techproducts/update/json?split=/exams'\ ---- ==== -V2 API SolrCloud:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/collections/techproducts/update/json?split=/exams'\ - -H 'Content-type:application/json' -d ' +curl 'http://localhost:8983/api/collections/techproducts/update/json'\ +'?split=/exams'\ + -H 'Content-Type: application/json' \ + -d ' { "first": "John", "last": "Doe", @@ -774,29 +611,18 @@ V1 API:: ==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update/json/docs' -H 'Content-type:application/json' -d ' -{ "first":"Steve", "last":"Jobs", "grade":1, "subject":"Social Science", "test":"term1", "marks":90} -{ "first":"Steve", "last":"Woz", "grade":1, "subject":"Political Science", "test":"term1", "marks":86}' ----- -==== - -V2 API User-Managed / Standalone Solr:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/collections/techproducts/update/json' -H 'Content-type:application/json' -d ' +curl 'http://localhost:8983/solr/techproducts/update/json/docs' -H 'Content-Type: application/json' --data-binary ' { "first":"Steve", "last":"Jobs", "grade":1, "subject":"Social Science", "test":"term1", "marks":90} { "first":"Steve", "last":"Woz", "grade":1, "subject":"Political Science", "test":"term1", "marks":86}' ---- ==== -V2 API SolrCloud:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/collections/techproducts/update/json' -H 'Content-type:application/json' -d ' +curl 'http://localhost:8983/api/collections/techproducts/update/json' -H 'Content-Type: application/json' --data-binary ' { "first":"Steve", "last":"Jobs", "grade":1, "subject":"Social Science", "test":"term1", "marks":90} { "first":"Steve", "last":"Woz", "grade":1, "subject":"Political Science", "test":"term1", "marks":86}' ---- @@ -812,29 +638,18 @@ V1 API:: ==== [source,bash] ---- -curl 'http://localhost:8983/solr/techproducts/update/json/docs' -H 'Content-type:application/json' -d '[ +curl 'http://localhost:8983/solr/techproducts/update/json/docs' -H 'Content-Type: application/json' --data-binary '[ {"first":"Steve", "last":"Jobs", "grade":1, "subject":"Computer Science", "test":"term1", "marks":90}, {"first":"Steve", "last":"Woz", "grade":1, "subject":"Calculus", "test":"term1", "marks":86}]' ---- ==== -V2 API User-Managed / Standalone Solr:: +V2 API:: + ==== [source,bash] ---- -curl 'http://localhost:8983/api/cores/techproducts/update/json' -H 'Content-type:application/json' -d '[ -{"first":"Steve", "last":"Jobs", "grade":1, "subject":"Computer Science", "test":"term1", "marks":90}, -{"first":"Steve", "last":"Woz", "grade":1, "subject":"Calculus", "test":"term1", "marks":86}]' ----- -==== - -V2 API SolrCloud:: -+ -==== -[source,bash] ----- -curl 'http://localhost:8983/api/collections/techproducts/update/json' -H 'Content-type:application/json' -d '[ +curl 'http://localhost:8983/api/collections/techproducts/update/json' -H 'Content-Type: application/json' --data-binary '[ {"first":"Steve", "last":"Jobs", "grade":1, "subject":"Computer Science", "test":"term1", "marks":90}, {"first":"Steve", "last":"Woz", "grade":1, "subject":"Calculus", "test":"term1", "marks":86}]' ---- @@ -854,7 +669,8 @@ Set the configuration as given in the Setting JSON Defaults section. == Setting JSON Defaults -It is possible to send any JSON to the `/update/json/docs` endpoint and the default configuration of the component is as follows: +It is possible to send any JSON to the V1 `/update/json/docs` endpoint or the V2 `/update/json` endpoint. +Both use the document loader configuration shown below: [source,xml] ---- @@ -884,22 +700,7 @@ V1 API:: ==== [source,bash] ---- - curl 'http://localhost:8983/solr/techproducts/config/params' -H 'Content-type:application/json' -d '{ -"set": { - "full_txt": { - "srcField": "_src_", - "mapUniqueKeyOnly" : true, - "df": "text" -}}}' ----- -==== - -V2 API User-Managed / Standalone Solr:: -+ -==== -[source,bash] ----- - curl 'http://localhost:8983/api/cores/techproducts/config/params' -H 'Content-type:application/json' -d '{ +curl 'http://localhost:8983/solr/techproducts/config/params' -H 'Content-Type: application/json' -d '{ "set": { "full_txt": { "srcField": "_src_", @@ -909,12 +710,12 @@ V2 API User-Managed / Standalone Solr:: ---- ==== -V2 API SolrCloud:: +V2 API:: + ==== [source,bash] ---- - curl 'http://localhost:8983/api/collections/techproducts/config/params' -H 'Content-type:application/json' -d '{ +curl 'http://localhost:8983/api/collections/techproducts/config/params' -H 'Content-Type: application/json' -d '{ "set": { "full_txt": { "srcField": "_src_", diff --git a/solr/solrj/src/resources/java-template/api.mustache b/solr/solrj/src/resources/java-template/api.mustache index b8c0dce81687..be772a1401fa 100644 --- a/solr/solrj/src/resources/java-template/api.mustache +++ b/solr/solrj/src/resources/java-template/api.mustache @@ -99,6 +99,9 @@ public class {{classname}} { {{/vendorExtensions.x-rawOutput}} {{#bodyParam}} private final {{{dataType}}} requestBody; + {{#vendorExtensions.x-genericEntity}} + private final String contentType; + {{/vendorExtensions.x-genericEntity}} {{/bodyParam}} {{#requiredParams}} {{^isBodyParam}} @@ -116,8 +119,14 @@ public class {{classname}} { * {{#requiredParams}}{{^isBodyParam}}* @param {{paramName}} Path param - {{description}}{{/isBodyParam}} {{/requiredParams}} + {{#bodyParam}}{{#vendorExtensions.x-genericEntity}} + * @param requestBody The request content, in the format given by {@code contentType}. + * @param contentType The MIME type of {@code requestBody} -- must be one of this + * operation's supported content types (e.g. "application/json"). The server + * selects how to interpret the body by this header, so it must be set accurately. + {{/vendorExtensions.x-genericEntity}}{{/bodyParam}} */ - public {{operationIdCamelCase}}({{#allParams}}{{#required}}{{^isBodyParam}}{{^-first}}, {{/-first}}{{{dataType}}} {{paramName}}{{/isBodyParam}}{{#isBodyParam}}{{#vendorExtensions.x-genericEntity}}{{^-first}}, {{/-first}}{{{dataType}}} requestBody{{/vendorExtensions.x-genericEntity}}{{/isBodyParam}}{{/required}}{{/allParams}}) { + public {{operationIdCamelCase}}({{#allParams}}{{#required}}{{^isBodyParam}}{{^-first}}, {{/-first}}{{{dataType}}} {{paramName}}{{/isBodyParam}}{{#isBodyParam}}{{#vendorExtensions.x-genericEntity}}{{^-first}}, {{/-first}}{{{dataType}}} requestBody, String contentType{{/vendorExtensions.x-genericEntity}}{{/isBodyParam}}{{/required}}{{/allParams}}) { // TODO Hardcode request type for now, but in reality we'll want to parse this out of the Operation data somehow super( SolrRequest.METHOD.valueOf("{{httpMethod}}"), @@ -130,6 +139,7 @@ public class {{classname}} { {{#isBodyParam}} {{#vendorExtensions.x-genericEntity}} this.requestBody = requestBody; + this.contentType = contentType; {{/vendorExtensions.x-genericEntity}} {{/isBodyParam}} {{^isBodyParam}} @@ -216,7 +226,7 @@ public class {{classname}} { @Override public String getContentType() { - return "application/octet-stream"; + return contentType; } }; {{/vendorExtensions.x-genericEntity}} diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java index ab0860f0b5e3..25f0754d7649 100644 --- a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java +++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleTests.java @@ -30,6 +30,8 @@ import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -40,6 +42,7 @@ import java.util.Locale; import java.util.Map; import java.util.Random; +import org.apache.commons.io.file.PathUtils; import org.apache.lucene.tests.util.TestUtil; import org.apache.solr.SolrTestCaseJ4.SuppressSSL; import org.apache.solr.client.solrj.embedded.EmbeddedSolrServer; @@ -57,6 +60,7 @@ import org.apache.solr.client.solrj.request.StreamingUpdateRequest; import org.apache.solr.client.solrj.request.SystemInfoRequest; import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.client.solrj.request.schema.SchemaRequest; import org.apache.solr.client.solrj.response.FacetField; import org.apache.solr.client.solrj.response.FieldStatsInfo; import org.apache.solr.client.solrj.response.InputStreamResponseParser; @@ -104,8 +108,16 @@ public abstract class SolrExampleTests extends SolrExampleTestsBase { public static void beforeTest() throws Exception { EnvUtils.setProperty( ALLOW_PATHS_SYSPROP, ExternalPaths.SERVER_HOME.toAbsolutePath().toString()); - solrTestRule.startSolr(); - solrTestRule.newCollection().withConfigSet(ExternalPaths.TECHPRODUCTS_CONFIGSET).create(); + Path solrHome = createTempDir("solrhome"); + Path configSet = solrHome.resolve("configsets/techproducts/conf"); + Files.createDirectories(configSet); + PathUtils.copyDirectory(ExternalPaths.TECHPRODUCTS_CONFIGSET, configSet); + solrTestRule.startSolr(solrHome); + solrTestRule.newCollection().withConfigSet(configSet).create(); + new SchemaRequest.DeleteField("_nest_path_") + .process(solrTestRule.getSolrClient(), DEFAULT_TEST_COLLECTION_NAME); + new SchemaRequest.DeleteField("_nest_parent_") + .process(solrTestRule.getSolrClient(), DEFAULT_TEST_COLLECTION_NAME); } @Test @@ -1038,6 +1050,8 @@ public void testLukeHandler() throws Exception { assertNumFound("*:*", doc.length); // make sure it got in LukeRequest luke = new LukeRequest(); + // LukeResponse expects the NamedList representation used by the binary response parser. + luke.setResponseParser(new JavaBinResponseParser()); luke.setShowSchema(false); LukeResponse rsp = luke.process(client); assertNull(rsp.getFieldTypeInfo()); // if you don't ask for it, the schema is null diff --git a/solr/webapp/src/test/org/apache/solr/webapp/AdminUiDocumentsScreenTest.java b/solr/webapp/src/test/org/apache/solr/webapp/AdminUiDocumentsScreenTest.java index 4ec9186a58db..66c0d6db7b14 100644 --- a/solr/webapp/src/test/org/apache/solr/webapp/AdminUiDocumentsScreenTest.java +++ b/solr/webapp/src/test/org/apache/solr/webapp/AdminUiDocumentsScreenTest.java @@ -44,6 +44,9 @@ public void testDocumentsScreenForm() { assertTrue("Doc type dropdown should offer JSON, got " + types, types.contains("JSON")); assertTrue("Doc type dropdown should offer XML, got " + types, types.contains("XML")); assertTrue("Doc type dropdown should offer CSV, got " + types, types.contains("CSV")); + assertTrue( + "Doc type dropdown should offer raw JSON commands, got " + types, + types.contains("Solr Command (raw JSON)")); waitFor(By.id("submit")); assertNoSevereConsoleErrors(); } diff --git a/solr/webapp/web/css/angular/documents.css b/solr/webapp/web/css/angular/documents.css index 2f0ba12ed7fd..4283b96741e9 100644 --- a/solr/webapp/web/css/angular/documents.css +++ b/solr/webapp/web/css/angular/documents.css @@ -27,7 +27,7 @@ limitations under the License. #content #documents #form { float: left; - /*width: 21%;*/ + width: 43%; } #content #documents #form label @@ -49,7 +49,8 @@ limitations under the License. #content #documents #form textarea { margin-bottom: 2px; - /*width: 98%;*/ + box-sizing: border-box; + max-width: 100%; } #content #documents #form #start @@ -123,6 +124,7 @@ limitations under the License. { float: right; width: 54%; + box-sizing: border-box; } #content #documents #result #url diff --git a/solr/webapp/web/js/angular/controllers/core-overview.js b/solr/webapp/web/js/angular/controllers/core-overview.js index d71eeed49a80..ad1fe8e011f2 100644 --- a/solr/webapp/web/js/angular/controllers/core-overview.js +++ b/solr/webapp/web/js/angular/controllers/core-overview.js @@ -23,7 +23,7 @@ var isTruthy = function(value) { }; solrAdminApp.controller('CoreOverviewController', -function($scope, $rootScope, $routeParams, $timeout, Luke, CoreInfo, Update, ReplicationV2, Ping, Constants, ApiErrorHandler) { +function($scope, $rootScope, $routeParams, $timeout, Luke, CoreInfo, ReplicationV2, Ping, Constants, ApiErrorHandler) { $scope.resetMenu("overview", Constants.IS_CORE_PAGE); $scope.refreshIndex = function() { Luke.index({core: $routeParams.core}, diff --git a/solr/webapp/web/js/angular/controllers/cores.js b/solr/webapp/web/js/angular/controllers/cores.js index 5a13e9ba244c..211c8cabdeb7 100644 --- a/solr/webapp/web/js/angular/controllers/cores.js +++ b/solr/webapp/web/js/angular/controllers/cores.js @@ -16,7 +16,7 @@ */ solrAdminApp.controller('CoreAdminController', - function($scope, $routeParams, $location, $timeout, $route, CoresV2, Update, Constants, ApiErrorHandler, NodeTarget){ + function($scope, $routeParams, $location, $timeout, $route, CoresV2, Constants, ApiErrorHandler, NodeTarget){ $scope.resetMenu("cores", Constants.IS_ROOT_PAGE); $scope.selectedCore = $routeParams.corename; // use 'corename' not 'core' to distinguish from /solr/:core/ $scope.refresh = function() { diff --git a/solr/webapp/web/js/angular/controllers/documents.js b/solr/webapp/web/js/angular/controllers/documents.js index d38265a05486..3629d12a9fa2 100644 --- a/solr/webapp/web/js/angular/controllers/documents.js +++ b/solr/webapp/web/js/angular/controllers/documents.js @@ -17,14 +17,25 @@ //helper for formatting JSON and others var DOC_PLACEHOLDER = '\n' + - 'change.me' + - 'change.me' + + ' change.me\n' + + ' change.me\n' + ''; -var ADD_PLACEHOLDER = '\n' + DOC_PLACEHOLDER + '\n'; +var JSON_DOC_PLACEHOLDER = '{\n' + + ' "id": "change.me",\n' + + ' "title": "change.me"\n' + + '}'; +var JSON_COMMAND_PLACEHOLDER = '{\n' + + ' "add": {\n' + + ' "doc": {\n' + + ' "id": "change.me",\n' + + ' "title": "change.me"\n' + + ' }\n' + + ' }\n' + + '}'; solrAdminApp.controller('DocumentsController', - function($scope, $rootScope, $routeParams, $location, Luke, Update, FileUpload, Constants) { + function($scope, $routeParams, Luke, UpdateV2, FileUpload, Constants, ApiErrorHandler) { $scope.resetMenu("documents", Constants.IS_COLLECTION_PAGE); $scope.refresh = function () { @@ -34,7 +45,6 @@ solrAdminApp.controller('DocumentsController', $scope.fields = Object.keys(data.schema.fields); }); $scope.document = ""; - $scope.handler = "/update"; $scope.type = "json"; $scope.commitWithin = 1000; $scope.overwrite = true; @@ -45,11 +55,11 @@ solrAdminApp.controller('DocumentsController', $scope.changeDocumentType = function () { $scope.placeholder = ""; if ($scope.type == 'json') { - $scope.placeholder = '{"id":"change.me","title":"change.me"}'; + $scope.placeholder = JSON_DOC_PLACEHOLDER; } else if ($scope.type == 'csv') { $scope.placeholder = "id,title\nchange.me,change.me"; - } else if ($scope.type == 'solr') { - $scope.placeholder = ADD_PLACEHOLDER; + } else if ($scope.type == 'solr-json') { + $scope.placeholder = JSON_COMMAND_PLACEHOLDER; } else if ($scope.type == 'xml') { $scope.placeholder = DOC_PLACEHOLDER; } @@ -64,74 +74,57 @@ solrAdminApp.controller('DocumentsController', }; $scope.submit = function () { - var contentType = ""; - var postData = ""; - var params = {}; - var doingFileUpload = false; - - if ($scope.handler[0] == '/') { - params.handler = $scope.handler.substring(1); - } else { - params.handler = 'update'; - params.qt = $scope.handler; + if ($scope.type == "upload") { + FileUpload.upload({ + core: $routeParams.core, + handler: "update", + commitWithin: $scope.commitWithin, + overwrite: $scope.overwrite, + wt: "json", + raw: $scope.literalParams + }, $scope.fileUpload, function (data) { + $scope.responseStatus = "success"; + $scope.response = JSON.stringify(data, null, ' '); + }, function (data) { + $scope.responseStatus = "failure"; + $scope.response = JSON.stringify(data, null, ' '); + }); + return; } - params.commitWithin = $scope.commitWithin; - params.overwrite = $scope.overwrite; - params.core = $routeParams.core; - params.wt = "json"; - + var postData; + var updateMethod; if ($scope.type == "json" || $scope.type == "wizard") { postData = "[" + $scope.document + "]"; - contentType = "json"; - } else if ($scope.type == "csv") { + updateMethod = UpdateV2.update; + } else if ($scope.type == "solr-json") { postData = $scope.document; - contentType = "csv"; + updateMethod = UpdateV2.update; } else if ($scope.type == "xml") { postData = "" + $scope.document + ""; - contentType = "xml"; - } else if ($scope.type == "upload") { - doingFileUpload = true; - params.raw = $scope.literalParams; - } else if ($scope.type == "solr") { + updateMethod = UpdateV2.updateXml; + } else if ($scope.type == "csv") { postData = $scope.document; - if (postData[0] == "<") { - contentType = "xml"; - } else if (postData[0] == "{" || postData[0] == '[') { - contentType = "json"; - } else { - alert("Cannot identify content type") - } + updateMethod = UpdateV2.updateCsv; } - if (!doingFileUpload) { - var callback = function (success) { - $scope.responseStatus = "success"; - delete success.$promise; - delete success.$resolved; - $scope.response = JSON.stringify(success, null, ' '); - }; - var failure = function (failure) { - $scope.responseStatus = failure; - }; - if (contentType == "json") { - Update.postJson(params, postData, callback, failure); - } else if (contentType == "xml") { - Update.postXml(params, postData, callback, failure); - } else if (contentType == "csv") { - Update.postCsv(params, postData, callback, failure); - } - } else { - var file = $scope.fileUpload; - console.log('file is ' + JSON.stringify(file)); - var uploadUrl = "/fileUpload"; - FileUpload.upload(params, $scope.fileUpload, function (success) { - $scope.responseStatus = "success"; - $scope.response = JSON.stringify(success, null, ' '); - }, function (failure) { + if (!updateMethod || $scope.isCloudEnabled === undefined) return; + + var indexType = $scope.isCloudEnabled ? "collections" : "cores"; + var updateOptions = { + commitWithin: $scope.commitWithin, + overwrite: $scope.overwrite + }; + var v2Callback = function (error, data, response) { + if (error) { $scope.responseStatus = "failure"; - $scope.response = JSON.stringify(failure, null, ' '); - }); - } + $scope.response = JSON.stringify((response && response.body) || error, null, ' '); + ApiErrorHandler.handle(response); + return; + } + $scope.responseStatus = "success"; + $scope.response = JSON.stringify(data, null, ' '); + $scope.$evalAsync(); + }; + updateMethod.call(UpdateV2, indexType, $routeParams.core, postData, updateOptions, v2Callback); } }); - diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js index 3fde3fd1dba9..acbdf8f6dccd 100644 --- a/solr/webapp/web/js/angular/services.js +++ b/solr/webapp/web/js/angular/services.js @@ -268,6 +268,12 @@ solrAdminServices.factory('Metrics', delete solrApi.ApiClient.instance.defaultHeaders["User-Agent"]; return new solrApi.SystemApi(); }) +.factory('UpdateV2', + function() { + solrApi.ApiClient.instance.basePath = '/api'; + delete solrApi.ApiClient.instance.defaultHeaders["User-Agent"]; + return new solrApi.UpdateApi(); + }) .factory('AliasesV2', function() { solrApi.ApiClient.instance.basePath = '/api'; @@ -402,16 +408,6 @@ solrAdminServices.factory('Metrics', ['$resource', function($resource) { return $resource(':core/admin/info', {wt:'json', core: "@core", _:Date.now()}); }]) -.factory('Update', - ['$resource', function($resource) { - return $resource(':core/:handler', {core: '@core', wt:'json', _:Date.now(), handler:'update'}, { - "commit": {params: {commit: "true"}}, - "post": {headers: {'Content-type': 'application/json'}, method: "POST", params: {handler: '@handler'}}, - "postJson": {headers: {'Content-type': 'application/json'}, method: "POST", params: {handler: '@handler'}}, - "postXml": {headers: {'Content-type': 'text/xml'}, method: "POST", params: {handler: '@handler'}}, - "postCsv": {headers: {'Content-type': 'application/csv'}, method: "POST", params: {handler: '@handler'}} - }); - }]) .factory('ParamSetsV2', function() { solrApi.ApiClient.instance.basePath = '/api'; diff --git a/solr/webapp/web/partials/documents.html b/solr/webapp/web/partials/documents.html index 2bf3f12982dc..29e26aaa7c53 100644 --- a/solr/webapp/web/partials/documents.html +++ b/solr/webapp/web/partials/documents.html @@ -20,10 +20,10 @@
- - +
+ Documents are submitted to the v2 endpoint + /api/{{isCloudEnabled ? 'collections' : 'cores'}}/{{currentCollection.name || currentCore.name}}/update. +
@@ -34,7 +34,7 @@ - +
@@ -107,5 +107,3 @@
- -