[CALCITE-7860] Add controls for remote fetches on file adapter

core/src/main/java/org/apache/calcite/config/CalciteSystemProperty.java

@@ -532,6 +532,31 @@ public final class CalciteSystemProperty<T> {
   public static final CalciteSystemProperty<String> MODEL_BASE_DIRECTORY =
       stringProperty("calcite.model.baseDirectory", "");
 
+  /**
+   * Regular expression that a remote URL in a file adapter table {@code url}
+   * operand must match for the adapter to dereference it (for example
+   * {@code -Dcalcite.file.urls.allowed=https://data\.example\.com/.*}).
+   * Empty by default: only local {@code file:} sources are dereferenced.
+   *
+   * <p>The expression uses {@link java.util.regex.Pattern} syntax and must
+   * match the whole URL, as written in the operand. Use {@code |} to allow
+   * several sites, for example
+   * {@code https://data\.example\.com/.*|http://data\.example\.com:8080/.*}.
+   * Matching is case-sensitive (use {@code (?i)} to ignore case), and the URL
+   * is not normalized first: default ports, {@code .} and {@code ..} segments
+   * and percent-encoding are compared as written. Escape each {@code .} in a
+   * host name, and end the host with {@code /} (or {@code :port/}),
+   * otherwise {@code example\.com.*} also matches {@code example.com.other.org}.
+   *
+   * <p>Only {@code file:} URLs that do not name a host (not even {@code localhost})
+   * are considered local and are always allowed. The check applies only to the URL
+   * named in the operand. Redirects returned by the server are followed without
+   * being re-checked, so an allowed URL that redirects can cause the adapter to
+   * fetch from a URL that does not match the expression.
+   */
+  public static final CalciteSystemProperty<String> FILE_URLS_ALLOWED =
+      stringProperty("calcite.file.urls.allowed", "");
+
   /**
    * Whether the Kafka adapter forwards the {@code consumer.params} model
    * operand to the Kafka client without filtering it.

file/src/main/java/org/apache/calcite/adapter/file/FileSchema.java

@@ -16,6 +16,7 @@
  */
 package org.apache.calcite.adapter.file;
 
+import org.apache.calcite.config.CalciteSystemProperty;
 import org.apache.calcite.schema.SchemaPlus;
 import org.apache.calcite.schema.Table;
 import org.apache.calcite.schema.impl.AbstractSchema;
@@ -25,18 +26,30 @@
 
 import au.com.bytecode.opencsv.CSVParser;
 
+import com.google.common.base.Suppliers;
 import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 
 import org.checkerframework.checker.nullness.qual.Nullable;
 
 import java.io.File;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
 import java.util.List;
 import java.util.Map;
+import java.util.function.Supplier;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
 
 /**
  * Schema mapped onto a set of URLs / HTML tables. Each table in the schema
  * is an HTML table on a URL.
+ *
+ * <p>By default, only local {@code file} sources are dereferenced. Table operands
+ * naming remote URLs make the host fetch an arbitrary, model-supplied URL and return the
+ * response as rows, so remote URLs must be explicitly allowed by the operator via the
+ * {@link CalciteSystemProperty#FILE_URLS_ALLOWED} system property.
  */
 class FileSchema extends AbstractSchema {
   private final ImmutableList<Map<String, Object>> tables;
@@ -119,10 +132,63 @@ private static String trim(String s, String suffix) {
     return builder.build();
   }
 
+  // Compile regex pattern for allowed remote urls just once
+  private static final @Nullable Supplier<Pattern> FILE_URLS_ALLOWED_PATTERN =
+      CalciteSystemProperty.FILE_URLS_ALLOWED.value().isEmpty()
+          ? null
+          : Suppliers.memoize(() -> {
+            String allowedUrls = CalciteSystemProperty.FILE_URLS_ALLOWED.value();
+            try {
+              return Pattern.compile(allowedUrls);
+            } catch (PatternSyntaxException e) {
+              throw new IllegalArgumentException("Invalid regular expression '" + allowedUrls
+                  + "' in system property \"calcite.file.urls.allowed\"", e);
+            }
+          });
+
+  /** Checks that a model-supplied table URL may be dereferenced by this
+   * adapter: a local {@code file} URL always, a remote one (any other
+   * protocol, or a {@code file} URL naming a host, such as a Windows UNC path)
+   * only when it matches the regular expression
+   * {@link CalciteSystemProperty#FILE_URLS_ALLOWED}. */
+  static void checkUrlAllowed(String url) {
+    checkUrlAllowed(url, FILE_URLS_ALLOWED_PATTERN);
+  }
+
+  static void checkUrlAllowed(String url, @Nullable Supplier<Pattern> pattern) {
+    final URI uri;
+    try {
+      uri = new URI(url);
+      uri.toURL();
+    } catch (URISyntaxException | MalformedURLException | IllegalArgumentException e) {
+      // Malformed; Sources.url parses the same way and will report the error
+      return;
+    }
+    if (isLocalUri(uri) || (pattern != null && pattern.get().matcher(url).matches())) {
+      return;
+    }
+    throw new IllegalArgumentException("Remote URL '" + url
+        + "' is not allowed, it does not match the \"calcite.file.urls.allowed\" pattern");
+  }
+
+  /**
+   * Returns whether a URI is local: a "file" URL is local unless it names a host,
+   * as an authority ("file://host/share") or, as Windows resolves it to a UNC path, as a path
+   * starting with two slashes or backslashes ("file:////host/share", "file:%5C%5Chost%5Cshare").
+   */
+  static boolean isLocalUri(URI uri) {
+    final String path =
+        Util.first(uri.isOpaque() ? uri.getSchemeSpecificPart() : uri.getPath(), "");
+    return "file".equalsIgnoreCase(uri.getScheme())
+        && Util.isNullOrEmpty(uri.getRawAuthority())
+        && !path.replace('\\', '/').startsWith("//");
+  }
+
   private boolean addTable(ImmutableMap.Builder<String, Table> builder,
       Map<String, Object> tableDef) {
     final String tableName = (String) tableDef.get("name");
     final String url = (String) tableDef.get("url");
+    checkUrlAllowed(url);
     final Source source0 = Sources.url(url);
     final Source source;
     if (baseDirectory == null) {

file/src/test/java/org/apache/calcite/adapter/file/FileSchemaProtocolTest.java

@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to you under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.calcite.adapter.file;
+
+import org.apache.calcite.config.CalciteSystemProperty;
+import org.apache.calcite.schema.Table;
+
+import com.google.common.collect.ImmutableList;
+
+import org.junit.jupiter.api.Test;
+
+import java.net.URI;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.regex.Pattern;
+
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+/**
+ * Tests that {@link FileSchema} dereferences only local {@code file} sources by
+ * default, and requires an explicit operator opt-in (the
+ * {@code calcite.file.urls.allowed} system property) for remote URL table
+ * operands.
+ */
+class FileSchemaProtocolTest {
+
+  private static Map<String, Object> table(String name, String url) {
+    final Map<String, Object> tableDef = new HashMap<>();
+    tableDef.put("name", name);
+    tableDef.put("url", url);
+    return tableDef;
+  }
+
+  /** Skips a test that relies on the default allowlist, if the test JVM was
+   * started with {@code calcite.file.urls.allowed} set. */
+  private static void assumeDefaultAllowlist() {
+    assumeTrue(CalciteSystemProperty.FILE_URLS_ALLOWED.value().isEmpty(),
+        "calcite.file.urls.allowed is set");
+  }
+
+  private static void assertAllowed(String url, String allowedUrls) {
+    assertDoesNotThrow(() -> checkUrlAllowed(url, allowedUrls),
+        () -> url + " should be allowed by " + allowedUrls);
+  }
+
+  private static void checkUrlAllowed(String url, String allowedUrls) {
+    FileSchema.checkUrlAllowed(url, () -> Pattern.compile(allowedUrls));
+  }
+
+  private static void assertRejected(String url, String allowedUrls) {
+    IllegalArgumentException e =
+        assertThrows(IllegalArgumentException.class,
+            () -> checkUrlAllowed(url, allowedUrls),
+            () -> url + " should be rejected by " + allowedUrls);
+    assertThat(url, e.getMessage(),
+        containsString("Remote URL '" + url + "' is not allowed"));
+    assertThat(url, e.getMessage(),
+        containsString("calcite.file.urls.allowed"));
+  }
+
+  @Test void testRemoteUrlRejectedByDefault() {
+    assumeDefaultAllowlist();
+    for (String remote : new String[] {"https://data.example.com/t.csv", "http://localhost/t.csv",
+        "ftp://localhost/t.csv", "file://fileserver.example.com/share/t.csv"}) {
+      final FileSchema schema =
+          new FileSchema(null, "S", null,
+              ImmutableList.of(table("T", remote)));
+      IllegalArgumentException e =
+          assertThrows(IllegalArgumentException.class, schema::getTableMap);
+      assertThat(e.getMessage(),
+          containsString("Remote URL '" + remote + "' is not allowed"));
+      assertThat(e.getMessage(), containsString("calcite.file.urls.allowed"));
+    }
+  }
+
+  @Test void testFileUrlAllowedByDefault() {
+    final FileSchema schema =
+        new FileSchema(null, "S", null,
+            ImmutableList.of(table("T", "file:foo.csv")));
+    final Map<String, Table> tableMap = schema.getTableMap();
+    assertThat(tableMap.containsKey("T"), is(true));
+  }
+
+  /** Tests that the regular expression must match the whole URL, not just a part of it. */
+  @Test void testRegexMatchesWholeUrl() {
+    assertRejected("https://data.example.com/table.csv",
+        "https://data\\.example\\.com/");
+    assertRejected("https://data.example.com/table.csv", "data\\.example\\.com");
+    assertRejected("https://other.example.com/?u=https://data.example.com/t.csv",
+        "https://data\\.example\\.com/.*");
+  }
+
+  @Test void testRegexAlternation() {
+    final String allowed =
+        "https://.*|http://data\\.example\\.com:8080/.*|ftp://ftp\\.example\\.com/.*";
+    assertAllowed("https://any.example.com:8443/x/table.csv", allowed);
+    assertAllowed("http://data.example.com:8080/table.csv", allowed);
+    assertAllowed("ftp://ftp.example.com/pub/table.csv", allowed);
+    assertRejected("http://data.example.com/table.csv", allowed);
+    assertRejected("ftp://other.example.com/pub/table.csv", allowed);
+  }
+
+  @Test void testIsLocalUri() throws Exception {
+    final String[] remoteUrls = {
+        "file://fileserver.example.com/share/table.csv",
+        "file:////fileserver.example.com/share/table.csv",
+        "file://///fileserver.example.com/share/table.csv",
+        "file:%5C%5Cfileserver.example.com%5Cshare%5Ctable.csv",
+        "file:/%5C%5Cfileserver.example.com/share/table.csv",
+        "file:%2F%2Fhost/share/t.csv",
+        "file://localhost/tmp/t.csv",
+        // jar URLs are considered as remote, even when they wrap a local file
+        "jar:file:/opt/app/lib/data.jar!/table.csv",
+        "jar:https://files.example.com/data.jar!/table.csv",
+    };
+    for (String url : remoteUrls) {
+      assertFalse(FileSchema.isLocalUri(new URI(url)), url);
+    }
+
+    final String[] localUrls = {
+        "file:table.csv",
+        "file:/tmp/table.csv",
+        "file:///tmp/table.csv",
+        "file:///C:/data/table.csv",
+        "file:/C:/data/table.csv",
+    };
+    for (String url : localUrls) {
+      assertTrue(FileSchema.isLocalUri(new URI(url)), url);
+    }
+  }
+}

site/_docs/file_adapter.md

@@ -27,6 +27,12 @@ limitations under the License.
 The file adapter is able to read files in a variety of formats,
 and can also read files over various protocols, such as HTTP.
 
+The file adapter dereferences only local `file:` sources by default. Table `url`
+operands naming remote protocols (`http`, `https`, `ftp`, `jar`, ...) or remote
+`file:` URLs (`file://host/...`) are
+supported only if the adapter is deployed with the appropriate system
+property, see [Enabling remote fetches](#enabling-remote-fetches).
+
 For example if you define:
 
 * States - https://en.wikipedia.org/wiki/List_of_states_and_territories_of_the_United_States
@@ -161,6 +167,9 @@ Now for a more complex example. This time we connect to Wikipedia via
 HTTP, read pages for US states and cities, and extract data from HTML
 tables on those pages. The tables have more complex formats, and the
 file adapter helps us locate and parse data in those tables.
+Note: since remote protocols are disabled by default (see
+[Enabling remote fetches](#enabling-remote-fetches)), run this example with
+`JAVA_OPTS='-Dcalcite.file.urls.allowed=https?://en\.wikipedia\.org/.*' ./sqlline`.
 
 Tables can be simply defined for immediate gratification:
 
@@ -344,6 +353,14 @@ sqlline> select distinct deptno from depts;
 3 rows selected (0.985 seconds)
 {% endhighlight %}
 
+## Enabling remote fetches
+
+From version 1.43.0 onwards, the file adapter dereferences only local `file:` sources by default.
+Table `url` operands naming remote URLs (`http`, `https`, `ftp`, `jar`, `file://host/`, ...) are
+rejected with `IllegalArgumentException` unless they match the regular expression specified by the
+[`calcite.file.urls.allowed`]({{ site.apiRoot }}/org/apache/calcite/config/CalciteSystemProperty.html#FILE_URLS_ALLOWED)
+system property.
+
 ## Future improvements
 
 We are continuing to enhance the adapter, and would welcome

site/_docs/history.md

@@ -114,6 +114,13 @@ check the width of each group, and turned `1-2-3-4-5` into
 requires exactly 16 bytes; a longer value used to be truncated. Blanks are not
 trimmed.
 
+* [<a href="https://issues.apache.org/jira/browse/CALCITE-7860">CALCITE-7860</a>]
+The file adapter now dereferences only local `file:` sources by default. Table
+`url` operands naming remote URLs (`http`, `https`, `ftp`, ...; including `file://localhost/`,
+and `jar:` even when it wraps a local file) are rejected with `IllegalArgumentException`
+unless they match the new `calcite.file.urls.allowed` system property. See
+[Enabling remote fetches]({{ site.baseurl }}/docs/file_adapter.html#enabling-remote-fetches).
+
 * [<a href="https://issues.apache.org/jira/browse/CALCITE-7760">CALCITE-7760</a>]
 The Spark engine now requires the operator-level opt-in system property
 `-Dcalcite.enable.spark=true`. The `spark` connection property alone no longer

site/_docs/security_threat_model.md

@@ -120,6 +120,17 @@ carve-out below. A report that reaches a sink not covered here is a model gap
   must add it on purpose.
 * A file, CSV, or JSON adapter reading the local path it was configured
   with. Opt-in, by the same reasoning as the os-adapter.
+* The file adapter dereferencing a remote URL (`http`, `https`, `ftp`, `jar`,
+  ..., or a `file:` URL naming a host, which Windows opens over SMB) named in a
+  table `url` operand, once the URL matches the regular expression that the
+  operator has set in the JVM system property `calcite.file.urls.allowed`. Only
+  local `file:` is dereferenced by default; a query author cannot set the property
+  (see [Attacker and trust boundary](#attacker-and-trust-boundary)), so any
+  fetch of an allowed URL is operator-authorised behaviour, not a P3
+  violation. This includes any redirect the remote server returns:
+  `calcite.file.urls.allowed` constrains only the URL in the operand,
+  not where the server redirects the request. Opt-in, by the same reasoning
+  as the os-adapter.
 * The Spark engine and its side effects: a local `JavaSparkContext`, a
   local HTTP class server that serves compiled query classes, and setting
   the `spark.repl.class.uri` JVM system property. The `spark` connection
@@ -168,6 +179,15 @@ vulnerability.
   references is configured and patched by the operator; its behavior past the
   connection boundary is out of this model.
 
+### Example modules
+
+Modules under `example/` (`example-csv`, `example-function`) are teaching
+artifacts referenced from the [tutorial]({{ site.baseurl }}/docs/tutorial.html),
+not production adapters. They are held to a lower bar than the rest of Calcite:
+a finding whose only reachable path is through an `example/` module, with no
+matching path in a production module, is a documentation issue rather than a
+vulnerability.
+
 ## Surprising vs unsurprising class loading
 
 Calcite loads a class named in a connection property or in SQL only to use it

site/_docs/tutorial.md

@@ -51,6 +51,12 @@ several important concepts:
 * advanced implementation of Table, using TranslatableTable, that translates
   to relational operators using planner rules.
 
+**Note:** Calcite-example-CSV is pedagogical code. It is intentionally minimal
+and does not carry the security hardening that ships in the production
+[file adapter]({{ site.baseurl }}/docs/file_adapter.html). Do not embed
+it in a service that exposes its `model` or JDBC connection properties to an
+untrusted query author; use `calcite-file` instead.
+
 ## Download and build
 
 You need Java (version 8, 9 or 10) and Git.