From 066ec8b8a92fcf8272fb393066309f2eba64981a Mon Sep 17 00:00:00 2001 From: Ruben Quesada Lopez Date: Fri, 16 Oct 2026 11:15:02 +0100 Subject: [PATCH] [CALCITE-7860] Add controls for remote fetches on file adapter --- .../calcite/config/CalciteSystemProperty.java | 25 +++ .../calcite/adapter/file/FileSchema.java | 66 ++++++++ .../adapter/file/FileSchemaProtocolTest.java | 152 ++++++++++++++++++ site/_docs/file_adapter.md | 17 ++ site/_docs/history.md | 7 + site/_docs/security_threat_model.md | 20 +++ site/_docs/tutorial.md | 6 + 7 files changed, 293 insertions(+) create mode 100644 file/src/test/java/org/apache/calcite/adapter/file/FileSchemaProtocolTest.java diff --git a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java index 39ba86b01150..8c89b03100e6 100644 --- a/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java +++ b/core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java @@ -532,6 +532,31 @@ public final class CalciteSystemProperty { public static final CalciteSystemProperty MODEL_BASE_DIRECTORY = stringProperty("calcite.model.baseDirectory", ""); + /** + * Regular expression that a remote URL in a file adapter table {@code url} + * operand must match for the adapter to dereference it (for example + * {@code -Dcalcite.file.urls.allowed=https://data\.example\.com/.*}). + * Empty by default: only local {@code file:} sources are dereferenced. + * + *

The expression uses {@link java.util.regex.Pattern} syntax and must + * match the whole URL, as written in the operand. Use {@code |} to allow + * several sites, for example + * {@code https://data\.example\.com/.*|http://data\.example\.com:8080/.*}. + * Matching is case-sensitive (use {@code (?i)} to ignore case), and the URL + * is not normalized first: default ports, {@code .} and {@code ..} segments + * and percent-encoding are compared as written. Escape each {@code .} in a + * host name, and end the host with {@code /} (or {@code :port/}), + * otherwise {@code example\.com.*} also matches {@code example.com.other.org}. + * + *

Only {@code file:} URLs that do not name a host (not even {@code localhost}) + * are considered local and are always allowed. The check applies only to the URL + * named in the operand. Redirects returned by the server are followed without + * being re-checked, so an allowed URL that redirects can cause the adapter to + * fetch from a URL that does not match the expression. + */ + public static final CalciteSystemProperty FILE_URLS_ALLOWED = + stringProperty("calcite.file.urls.allowed", ""); + /** * Whether the Kafka adapter forwards the {@code consumer.params} model * operand to the Kafka client without filtering it. diff --git a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java index f66effb4a32e..9298f3f9e8ab 100644 --- a/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java +++ b/file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java @@ -16,6 +16,7 @@ */ package org.apache.calcite.adapter.file; +import org.apache.calcite.config.CalciteSystemProperty; import org.apache.calcite.schema.SchemaPlus; import org.apache.calcite.schema.Table; import org.apache.calcite.schema.impl.AbstractSchema; @@ -25,18 +26,30 @@ import au.com.bytecode.opencsv.CSVParser; +import com.google.common.base.Suppliers; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.checkerframework.checker.nullness.qual.Nullable; import java.io.File; +import java.net.MalformedURLException; +import java.net.URI; +import java.net.URISyntaxException; import java.util.List; import java.util.Map; +import java.util.function.Supplier; +import java.util.regex.Pattern; +import java.util.regex.PatternSyntaxException; /** * Schema mapped onto a set of URLs / HTML tables. Each table in the schema * is an HTML table on a URL. + * + *

By default, only local {@code file} sources are dereferenced. Table operands + * naming remote URLs make the host fetch an arbitrary, model-supplied URL and return the + * response as rows, so remote URLs must be explicitly allowed by the operator via the + * {@link CalciteSystemProperty#FILE_URLS_ALLOWED} system property. */ class FileSchema extends AbstractSchema { private final ImmutableList> tables; @@ -119,10 +132,63 @@ private static String trim(String s, String suffix) { return builder.build(); } + // Compile regex pattern for allowed remote urls just once + private static final @Nullable Supplier FILE_URLS_ALLOWED_PATTERN = + CalciteSystemProperty.FILE_URLS_ALLOWED.value().isEmpty() + ? null + : Suppliers.memoize(() -> { + String allowedUrls = CalciteSystemProperty.FILE_URLS_ALLOWED.value(); + try { + return Pattern.compile(allowedUrls); + } catch (PatternSyntaxException e) { + throw new IllegalArgumentException("Invalid regular expression '" + allowedUrls + + "' in system property \"calcite.file.urls.allowed\"", e); + } + }); + + /** Checks that a model-supplied table URL may be dereferenced by this + * adapter: a local {@code file} URL always, a remote one (any other + * protocol, or a {@code file} URL naming a host, such as a Windows UNC path) + * only when it matches the regular expression + * {@link CalciteSystemProperty#FILE_URLS_ALLOWED}. */ + static void checkUrlAllowed(String url) { + checkUrlAllowed(url, FILE_URLS_ALLOWED_PATTERN); + } + + static void checkUrlAllowed(String url, @Nullable Supplier pattern) { + final URI uri; + try { + uri = new URI(url); + uri.toURL(); + } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) { + // Malformed; Sources.url parses the same way and will report the error + return; + } + if (isLocalUri(uri) || (pattern != null && pattern.get().matcher(url).matches())) { + return; + } + throw new IllegalArgumentException("Remote URL '" + url + + "' is not allowed, it does not match the \"calcite.file.urls.allowed\" pattern"); + } + + /** + * Returns whether a URI is local: a "file" URL is local unless it names a host, + * as an authority ("file://host/share") or, as Windows resolves it to a UNC path, as a path + * starting with two slashes or backslashes ("file:////host/share", "file:%5C%5Chost%5Cshare"). + */ + static boolean isLocalUri(URI uri) { + final String path = + Util.first(uri.isOpaque() ? uri.getSchemeSpecificPart() : uri.getPath(), ""); + return "file".equalsIgnoreCase(uri.getScheme()) + && Util.isNullOrEmpty(uri.getRawAuthority()) + && !path.replace('\\', '/').startsWith("//"); + } + private boolean addTable(ImmutableMap.Builder builder, Map tableDef) { final String tableName = (String) tableDef.get("name"); final String url = (String) tableDef.get("url"); + checkUrlAllowed(url); final Source source0 = Sources.url(url); final Source source; if (baseDirectory == null) { diff --git a/file/src/test/java/org/apache/calcite/adapter/file/FileSchemaProtocolTest.java b/file/src/test/java/org/apache/calcite/adapter/file/FileSchemaProtocolTest.java new file mode 100644 index 000000000000..1121fb04e63f --- /dev/null +++ b/file/src/test/java/org/apache/calcite/adapter/file/FileSchemaProtocolTest.java @@ -0,0 +1,152 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to you under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.calcite.adapter.file; + +import org.apache.calcite.config.CalciteSystemProperty; +import org.apache.calcite.schema.Table; + +import com.google.common.collect.ImmutableList; + +import org.junit.jupiter.api.Test; + +import java.net.URI; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Pattern; + +import static org.hamcrest.CoreMatchers.containsString; +import static org.hamcrest.CoreMatchers.is; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.junit.jupiter.api.Assertions.assertDoesNotThrow; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Tests that {@link FileSchema} dereferences only local {@code file} sources by + * default, and requires an explicit operator opt-in (the + * {@code calcite.file.urls.allowed} system property) for remote URL table + * operands. + */ +class FileSchemaProtocolTest { + + private static Map table(String name, String url) { + final Map tableDef = new HashMap<>(); + tableDef.put("name", name); + tableDef.put("url", url); + return tableDef; + } + + /** Skips a test that relies on the default allowlist, if the test JVM was + * started with {@code calcite.file.urls.allowed} set. */ + private static void assumeDefaultAllowlist() { + assumeTrue(CalciteSystemProperty.FILE_URLS_ALLOWED.value().isEmpty(), + "calcite.file.urls.allowed is set"); + } + + private static void assertAllowed(String url, String allowedUrls) { + assertDoesNotThrow(() -> checkUrlAllowed(url, allowedUrls), + () -> url + " should be allowed by " + allowedUrls); + } + + private static void checkUrlAllowed(String url, String allowedUrls) { + FileSchema.checkUrlAllowed(url, () -> Pattern.compile(allowedUrls)); + } + + private static void assertRejected(String url, String allowedUrls) { + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, + () -> checkUrlAllowed(url, allowedUrls), + () -> url + " should be rejected by " + allowedUrls); + assertThat(url, e.getMessage(), + containsString("Remote URL '" + url + "' is not allowed")); + assertThat(url, e.getMessage(), + containsString("calcite.file.urls.allowed")); + } + + @Test void testRemoteUrlRejectedByDefault() { + assumeDefaultAllowlist(); + for (String remote : new String[] {"https://data.example.com/t.csv", "http://localhost/t.csv", + "ftp://localhost/t.csv", "file://fileserver.example.com/share/t.csv"}) { + final FileSchema schema = + new FileSchema(null, "S", null, + ImmutableList.of(table("T", remote))); + IllegalArgumentException e = + assertThrows(IllegalArgumentException.class, schema::getTableMap); + assertThat(e.getMessage(), + containsString("Remote URL '" + remote + "' is not allowed")); + assertThat(e.getMessage(), containsString("calcite.file.urls.allowed")); + } + } + + @Test void testFileUrlAllowedByDefault() { + final FileSchema schema = + new FileSchema(null, "S", null, + ImmutableList.of(table("T", "file:foo.csv"))); + final Map tableMap = schema.getTableMap(); + assertThat(tableMap.containsKey("T"), is(true)); + } + + /** Tests that the regular expression must match the whole URL, not just a part of it. */ + @Test void testRegexMatchesWholeUrl() { + assertRejected("https://data.example.com/table.csv", + "https://data\\.example\\.com/"); + assertRejected("https://data.example.com/table.csv", "data\\.example\\.com"); + assertRejected("https://other.example.com/?u=https://data.example.com/t.csv", + "https://data\\.example\\.com/.*"); + } + + @Test void testRegexAlternation() { + final String allowed = + "https://.*|http://data\\.example\\.com:8080/.*|ftp://ftp\\.example\\.com/.*"; + assertAllowed("https://any.example.com:8443/x/table.csv", allowed); + assertAllowed("http://data.example.com:8080/table.csv", allowed); + assertAllowed("ftp://ftp.example.com/pub/table.csv", allowed); + assertRejected("http://data.example.com/table.csv", allowed); + assertRejected("ftp://other.example.com/pub/table.csv", allowed); + } + + @Test void testIsLocalUri() throws Exception { + final String[] remoteUrls = { + "file://fileserver.example.com/share/table.csv", + "file:////fileserver.example.com/share/table.csv", + "file://///fileserver.example.com/share/table.csv", + "file:%5C%5Cfileserver.example.com%5Cshare%5Ctable.csv", + "file:/%5C%5Cfileserver.example.com/share/table.csv", + "file:%2F%2Fhost/share/t.csv", + "file://localhost/tmp/t.csv", + // jar URLs are considered as remote, even when they wrap a local file + "jar:file:/opt/app/lib/data.jar!/table.csv", + "jar:https://files.example.com/data.jar!/table.csv", + }; + for (String url : remoteUrls) { + assertFalse(FileSchema.isLocalUri(new URI(url)), url); + } + + final String[] localUrls = { + "file:table.csv", + "file:/tmp/table.csv", + "file:///tmp/table.csv", + "file:///C:/data/table.csv", + "file:/C:/data/table.csv", + }; + for (String url : localUrls) { + assertTrue(FileSchema.isLocalUri(new URI(url)), url); + } + } +} diff --git a/site/_docs/file_adapter.md b/site/_docs/file_adapter.md index dc31fec4134c..e046b9f3f036 100644 --- a/site/_docs/file_adapter.md +++ b/site/_docs/file_adapter.md @@ -27,6 +27,12 @@ limitations under the License. The file adapter is able to read files in a variety of formats, and can also read files over various protocols, such as HTTP. +The file adapter dereferences only local `file:` sources by default. Table `url` +operands naming remote protocols (`http`, `https`, `ftp`, `jar`, ...) or remote +`file:` URLs (`file://host/...`) are +supported only if the adapter is deployed with the appropriate system +property, see [Enabling remote fetches](#enabling-remote-fetches). + For example if you define: * States - https://en.wikipedia.org/wiki/List_of_states_and_territories_of_the_United_States @@ -161,6 +167,9 @@ Now for a more complex example. This time we connect to Wikipedia via HTTP, read pages for US states and cities, and extract data from HTML tables on those pages. The tables have more complex formats, and the file adapter helps us locate and parse data in those tables. +Note: since remote protocols are disabled by default (see +[Enabling remote fetches](#enabling-remote-fetches)), run this example with +`JAVA_OPTS='-Dcalcite.file.urls.allowed=https?://en\.wikipedia\.org/.*' ./sqlline`. Tables can be simply defined for immediate gratification: @@ -344,6 +353,14 @@ sqlline> select distinct deptno from depts; 3 rows selected (0.985 seconds) {% endhighlight %} +## Enabling remote fetches + +From version 1.43.0 onwards, the file adapter dereferences only local `file:` sources by default. +Table `url` operands naming remote URLs (`http`, `https`, `ftp`, `jar`, `file://host/`, ...) are +rejected with `IllegalArgumentException` unless they match the regular expression specified by the +[`calcite.file.urls.allowed`]({{ site.apiRoot }}/org/apache/calcite/config/CalciteSystemProperty.html#FILE_URLS_ALLOWED) +system property. + ## Future improvements We are continuing to enhance the adapter, and would welcome diff --git a/site/_docs/history.md b/site/_docs/history.md index 0b6d23450b61..1fea0c9f1fbb 100644 --- a/site/_docs/history.md +++ b/site/_docs/history.md @@ -114,6 +114,13 @@ check the width of each group, and turned `1-2-3-4-5` into requires exactly 16 bytes; a longer value used to be truncated. Blanks are not trimmed. +* [CALCITE-7860] +The file adapter now dereferences only local `file:` sources by default. Table +`url` operands naming remote URLs (`http`, `https`, `ftp`, ...; including `file://localhost/`, +and `jar:` even when it wraps a local file) are rejected with `IllegalArgumentException` +unless they match the new `calcite.file.urls.allowed` system property. See +[Enabling remote fetches]({{ site.baseurl }}/docs/file_adapter.html#enabling-remote-fetches). + * [CALCITE-7760] The Spark engine now requires the operator-level opt-in system property `-Dcalcite.enable.spark=true`. The `spark` connection property alone no longer diff --git a/site/_docs/security_threat_model.md b/site/_docs/security_threat_model.md index 1b5e72493b09..c8ec88dcbb54 100644 --- a/site/_docs/security_threat_model.md +++ b/site/_docs/security_threat_model.md @@ -120,6 +120,17 @@ carve-out below. A report that reaches a sink not covered here is a model gap must add it on purpose. * A file, CSV, or JSON adapter reading the local path it was configured with. Opt-in, by the same reasoning as the os-adapter. +* The file adapter dereferencing a remote URL (`http`, `https`, `ftp`, `jar`, + ..., or a `file:` URL naming a host, which Windows opens over SMB) named in a + table `url` operand, once the URL matches the regular expression that the + operator has set in the JVM system property `calcite.file.urls.allowed`. Only + local `file:` is dereferenced by default; a query author cannot set the property + (see [Attacker and trust boundary](#attacker-and-trust-boundary)), so any + fetch of an allowed URL is operator-authorised behaviour, not a P3 + violation. This includes any redirect the remote server returns: + `calcite.file.urls.allowed` constrains only the URL in the operand, + not where the server redirects the request. Opt-in, by the same reasoning + as the os-adapter. * The Spark engine and its side effects: a local `JavaSparkContext`, a local HTTP class server that serves compiled query classes, and setting the `spark.repl.class.uri` JVM system property. The `spark` connection @@ -168,6 +179,15 @@ vulnerability. references is configured and patched by the operator; its behavior past the connection boundary is out of this model. +### Example modules + +Modules under `example/` (`example-csv`, `example-function`) are teaching +artifacts referenced from the [tutorial]({{ site.baseurl }}/docs/tutorial.html), +not production adapters. They are held to a lower bar than the rest of Calcite: +a finding whose only reachable path is through an `example/` module, with no +matching path in a production module, is a documentation issue rather than a +vulnerability. + ## Surprising vs unsurprising class loading Calcite loads a class named in a connection property or in SQL only to use it diff --git a/site/_docs/tutorial.md b/site/_docs/tutorial.md index 9ca5c02199d9..a4f0cc6d0b2b 100644 --- a/site/_docs/tutorial.md +++ b/site/_docs/tutorial.md @@ -51,6 +51,12 @@ several important concepts: * advanced implementation of Table, using TranslatableTable, that translates to relational operators using planner rules. +**Note:** Calcite-example-CSV is pedagogical code. It is intentionally minimal +and does not carry the security hardening that ships in the production +[file adapter]({{ site.baseurl }}/docs/file_adapter.html). Do not embed +it in a service that exposes its `model` or JDBC connection properties to an +untrusted query author; use `calcite-file` instead. + ## Download and build You need Java (version 8, 9 or 10) and Git.