diff --git a/.github/references/docker_manifest.json b/.github/references/docker_manifest.json
index c11f4cd4d2..64de854b72 100644
--- a/.github/references/docker_manifest.json
+++ b/.github/references/docker_manifest.json
@@ -506,7 +506,7 @@
},
"Proxy Host": {
"name": "Proxy Host",
- "description": "Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead.",
+ "description": "Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead.",
"validator": "VALID",
"required": "false",
"sensitive": "false",
@@ -853,7 +853,7 @@
},
"Proxy Host": {
"name": "Proxy Host",
- "description": "Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead.",
+ "description": "Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead.",
"validator": "VALID",
"required": "false",
"sensitive": "false",
@@ -1226,7 +1226,7 @@
},
"Proxy Host": {
"name": "Proxy Host",
- "description": "Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead.",
+ "description": "Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead.",
"validator": "VALID",
"required": "false",
"sensitive": "false",
@@ -1615,7 +1615,7 @@
},
"Proxy Host": {
"name": "Proxy Host",
- "description": "Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead.",
+ "description": "Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead.",
"validator": "VALID",
"required": "false",
"sensitive": "false",
@@ -2046,7 +2046,7 @@
},
"Proxy Host": {
"name": "Proxy Host",
- "description": "Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead.",
+ "description": "Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead.",
"validator": "VALID",
"required": "false",
"sensitive": "false",
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bce44bf299..ccb4ae88db 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -575,6 +575,10 @@ jobs:
sudo rm -rf /opt/ghc
sudo rm -rf "/usr/local/share/boost"
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
+ - id: dry_run
+ name: Behave dry run
+ working-directory: ./build
+ run: make docker-verify-modular-dry-run
- id: test
name: Docker Verify
working-directory: ./build
@@ -622,6 +626,10 @@ jobs:
run: |
sudo apt update
sudo apt install -y python3-virtualenv
+ - id: dry_run
+ name: Behave dry run
+ working-directory: ./build
+ run: ../docker/DryRunBehaveTests.sh ../minifi_rust
- id: test
name: Docker Verify
working-directory: ./minifi_rust
diff --git a/PROCESSORS.md b/PROCESSORS.md
index 4319a369ce..ef442fc21d 100644
--- a/PROCESSORS.md
+++ b/PROCESSORS.md
@@ -675,7 +675,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| **Region** | us-west-2 | af-south-1
ap-east-1
ap-east-2
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-south-2
ap-southeast-1
ap-southeast-2
ap-southeast-3
ap-southeast-4
ap-southeast-5
ap-southeast-6
ap-southeast-7
ca-central-1
ca-west-1
cn-north-1
cn-northwest-1
eu-central-1
eu-central-2
eu-isoe-west-1
eu-north-1
eu-south-1
eu-south-2
eu-west-1
eu-west-2
eu-west-3
eusc-de-east-1
il-central-1
me-central-1
me-south-1
mx-central-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-isof-east-1
us-isob-west-1
us-isof-south-1
us-west-1
us-west-2 | AWS Region |
| **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client |
| Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints. |
-| Proxy Host | | | Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead. |
+| Proxy Host | | | Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Port | | | The port number of the proxy host. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Username | | | Username to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Password | | | Password to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead.
**Sensitive Property: true** |
@@ -1163,7 +1163,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| **Region** | us-west-2 | af-south-1
ap-east-1
ap-east-2
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-south-2
ap-southeast-1
ap-southeast-2
ap-southeast-3
ap-southeast-4
ap-southeast-5
ap-southeast-6
ap-southeast-7
ca-central-1
ca-west-1
cn-north-1
cn-northwest-1
eu-central-1
eu-central-2
eu-isoe-west-1
eu-north-1
eu-south-1
eu-south-2
eu-west-1
eu-west-2
eu-west-3
eusc-de-east-1
il-central-1
me-central-1
me-south-1
mx-central-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-isof-east-1
us-isob-west-1
us-isof-south-1
us-west-1
us-west-2 | AWS Region |
| **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client |
| Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints. |
-| Proxy Host | | | Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead. |
+| Proxy Host | | | Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Port | | | The port number of the proxy host. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Username | | | Username to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Password | | | Password to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead.
**Sensitive Property: true** |
@@ -1842,7 +1842,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| **Region** | us-west-2 | af-south-1
ap-east-1
ap-east-2
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-south-2
ap-southeast-1
ap-southeast-2
ap-southeast-3
ap-southeast-4
ap-southeast-5
ap-southeast-6
ap-southeast-7
ca-central-1
ca-west-1
cn-north-1
cn-northwest-1
eu-central-1
eu-central-2
eu-isoe-west-1
eu-north-1
eu-south-1
eu-south-2
eu-west-1
eu-west-2
eu-west-3
eusc-de-east-1
il-central-1
me-central-1
me-south-1
mx-central-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-isof-east-1
us-isob-west-1
us-isof-south-1
us-west-1
us-west-2 | AWS Region |
| **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client |
| Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints. |
-| Proxy Host | | | Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead. |
+| Proxy Host | | | Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Port | | | The port number of the proxy host. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Username | | | Username to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Password | | | Password to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead.
**Sensitive Property: true** |
@@ -2742,7 +2742,7 @@ In the list below, the names of required properties appear in bold. Any other pr
| **Region** | us-west-2 | af-south-1
ap-east-1
ap-east-2
ap-northeast-1
ap-northeast-2
ap-northeast-3
ap-south-1
ap-south-2
ap-southeast-1
ap-southeast-2
ap-southeast-3
ap-southeast-4
ap-southeast-5
ap-southeast-6
ap-southeast-7
ca-central-1
ca-west-1
cn-north-1
cn-northwest-1
eu-central-1
eu-central-2
eu-isoe-west-1
eu-north-1
eu-south-1
eu-south-2
eu-west-1
eu-west-2
eu-west-3
eusc-de-east-1
il-central-1
me-central-1
me-south-1
mx-central-1
sa-east-1
us-east-1
us-east-2
us-gov-east-1
us-gov-west-1
us-iso-east-1
us-iso-west-1
us-isob-east-1
us-isof-east-1
us-isob-west-1
us-isof-south-1
us-west-1
us-west-2 | AWS Region |
| **Communications Timeout** | 30 sec | | Sets the timeout of the communication between the AWS server and the client |
| Endpoint Override URL | | | Endpoint URL to use instead of the AWS default including scheme, host, port, and path. The AWS libraries select an endpoint URL based on the AWS region, but this property overrides the selected endpoint URL, allowing use with other S3-compatible endpoints. |
-| Proxy Host | | | Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead. |
+| Proxy Host | | | Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Port | | | The port number of the proxy host. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Username | | | Username to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead. |
| Proxy Password | | | Password to set when authenticating against proxy. DEPRECATED, please use Proxy Configuration Service instead.
**Sensitive Property: true** |
diff --git a/cmake/DockerConfig.cmake b/cmake/DockerConfig.cmake
index e54da6750b..14d5f2bf6b 100644
--- a/cmake/DockerConfig.cmake
+++ b/cmake/DockerConfig.cmake
@@ -150,6 +150,10 @@ add_custom_target(
docker-verify-modular
COMMAND ${CMAKE_SOURCE_DIR}/docker/RunBehaveTests.sh ${PROJECT_VERSION_STR} ${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} --parallel_processes=${DOCKER_VERIFY_THREADS})
+add_custom_target(
+ docker-verify-modular-dry-run
+ COMMAND ${CMAKE_SOURCE_DIR}/docker/DryRunBehaveTests.sh --tags_to_run=${ENABLED_TAGS} --tags_to_exclude=${DISABLED_TAGS} ${CMAKE_SOURCE_DIR}/extensions ${CMAKE_SOURCE_DIR}/minifi_rust/extensions)
+
function(CREATE_DOCKER_TARGET_FROM_ROCKY_PACKAGE BASE_IMAGE TAG_PREFIX INSTALL_PACKAGE_CMD)
add_custom_target(
${TAG_PREFIX}_from_rocky_package
diff --git a/docker/DryRunBehaveTests.sh b/docker/DryRunBehaveTests.sh
new file mode 100755
index 0000000000..f2184e51c3
--- /dev/null
+++ b/docker/DryRunBehaveTests.sh
@@ -0,0 +1,129 @@
+#!/bin/bash
+
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -e
+
+print_help()
+{
+ printf '%s\n' "Dry runs the behave feature directories found in the provided directories"
+ printf '%s\n' "to catch environment.py and step module loading errors, which are silently skipped by behavex"
+ printf 'Usage: %s [--tags_to_run=] [--tags_to_exclude=] [-h|--help] ...\n' "$0"
+ printf '\t%s\n' ": directory to search for feature files recursively"
+ printf '\t%s\n' "--tags_to_run: optional comma-separated list of tags to include, only the feature directories having features with any of these tags are checked (default: all)"
+ printf '\t%s\n' "--tags_to_exclude: optional comma-separated list of tags that should be skipped (default: none)"
+ printf '\t%s\n' "-h, --help: Prints help"
+}
+
+tags_to_run=
+tags_to_exclude=
+search_dirs=()
+for arg in "$@"; do
+ case "${arg}" in
+ --tags_to_run=*)
+ tags_to_run="${arg##--tags_to_run=}"
+ ;;
+ --tags_to_exclude=*)
+ tags_to_exclude="${arg##--tags_to_exclude=}"
+ ;;
+ -h|--help)
+ print_help
+ exit 0
+ ;;
+ *)
+ search_dirs+=("${arg}")
+ ;;
+ esac
+done
+
+if [ "${#search_dirs[@]}" -eq 0 ]; then
+ print_help >&2
+ exit 1
+fi
+
+root_dir="$( cd "${0%/*}/.." && pwd )"
+
+# Create virtual environment for testing
+if [[ ! -d ./behave_venv ]]; then
+ echo "Creating virtual environment in ./behave_venv" 1>&2
+ virtualenv --python=python3 ./behave_venv
+fi
+
+echo "Activating virtual environment..." 1>&2
+# shellcheck disable=SC1091
+. ./behave_venv/bin/activate
+pip install -e "${root_dir}/behave_framework"
+
+TAG_OPTS=()
+if [ -n "${tags_to_run}" ]; then
+ TAG_OPTS+=(-t "${tags_to_run}")
+fi
+IFS=',' read -ra excluded_tags <<< "${tags_to_exclude}"
+for tag in "${excluded_tags[@]}"; do
+ TAG_OPTS+=(-t "~${tag}")
+done
+
+FEATURE_FILES=()
+while IFS= read -r -d '' file; do
+ FEATURE_FILES+=("${file}")
+done < <(find "${search_dirs[@]}" -name .venv -prune -o -type f -name '*.feature' -print0)
+
+if [ "${#FEATURE_FILES[@]}" -eq 0 ]; then
+ echo "No feature files found in: ${search_dirs[*]}" >&2
+ exit 1
+fi
+
+# Scenario Outlines without example rows are silently dropped by behave
+python3 -I - "${FEATURE_FILES[@]}" <<'EOF'
+import sys
+from behave.model import ScenarioOutline
+from behave.parser import parse_file
+
+empty_outlines = []
+for feature_file in sys.argv[1:]:
+ for scenario in parse_file(feature_file).walk_scenarios(with_outlines=True):
+ if isinstance(scenario, ScenarioOutline) and not any(examples.table and examples.table.rows for examples in scenario.examples):
+ empty_outlines.append(f"{feature_file}:{scenario.line}: {scenario.name}")
+
+if empty_outlines:
+ print("Scenario Outlines without examples found, use Scenario instead:", file=sys.stderr)
+ print("\n".join(f" {outline}" for outline in empty_outlines), file=sys.stderr)
+ sys.exit(1)
+EOF
+
+FEATURE_DIRS=()
+while IFS= read -r dir; do
+ FEATURE_DIRS+=("${dir}")
+done < <(
+ if [ -n "${tags_to_run}" ]; then
+ grep -lE "@(${tags_to_run//,/|})([[:space:]]|$)" "${FEATURE_FILES[@]}"
+ else
+ printf '%s\n' "${FEATURE_FILES[@]}"
+ fi | sed 's|/[^/]*$||' | sort -u)
+
+failed_dirs=()
+for dir in "${FEATURE_DIRS[@]}"; do
+ echo "Dry run of ${dir}" 1>&2
+ behave --dry-run --no-summary --format null "${TAG_OPTS[@]}" "${dir}" || failed_dirs+=("${dir}")
+done
+
+if [ "${#failed_dirs[@]}" -gt 0 ]; then
+ echo "Behave dry run failed, the following feature directories could not be loaded:" >&2
+ printf ' %s\n' "${failed_dirs[@]}" >&2
+ exit 1
+fi
+
+echo "Behave dry run succeeded for ${#FEATURE_DIRS[@]} feature directories" 1>&2
diff --git a/extensions/aws/processors/AwsProcessor.cpp b/extensions/aws/processors/AwsProcessor.cpp
index 979ba14509..23b70259a5 100644
--- a/extensions/aws/processors/AwsProcessor.cpp
+++ b/extensions/aws/processors/AwsProcessor.cpp
@@ -120,9 +120,15 @@ void AwsProcessor::onSchedule(core::ProcessContext& context, core::ProcessSessio
auto proxy = getProxy(context);
if (proxy.proxy_type != minifi::controllers::ProxyType::DIRECT) {
- if (minifi::utils::string::startsWith(proxy.proxy_host, "https://")) {
+ static constexpr std::string_view HTTPS_PREFIX = "https://";
+ if (minifi::utils::string::startsWith(proxy.proxy_host, HTTPS_PREFIX, false)) {
throw Exception(PROCESS_SCHEDULE_EXCEPTION, "HTTPS proxy is not supported");
}
+ // the AWS CRT client expects a plain host name without a scheme
+ static constexpr std::string_view HTTP_PREFIX = "http://";
+ if (minifi::utils::string::startsWith(proxy.proxy_host, HTTP_PREFIX, false)) {
+ proxy.proxy_host.erase(0, HTTP_PREFIX.size());
+ }
client_config_.proxyScheme = Aws::Http::Scheme::HTTP;
client_config_.proxyHost = proxy.proxy_host;
client_config_.proxyPort = proxy.proxy_port;
diff --git a/extensions/aws/processors/AwsProcessor.h b/extensions/aws/processors/AwsProcessor.h
index c06c14b7b8..9873a8dfb9 100644
--- a/extensions/aws/processors/AwsProcessor.h
+++ b/extensions/aws/processors/AwsProcessor.h
@@ -130,7 +130,7 @@ class AwsProcessor : public core::ProcessorImpl { // NOLINT(cppcoreguidelines-s
.withValidator(core::StandardPropertyValidators::NON_BLANK_VALIDATOR)
.build();
EXTENSIONAPI static constexpr auto ProxyHost = core::PropertyDefinitionBuilder<>::createProperty("Proxy Host")
- .withDescription("Proxy host name or IP. Use https:// prefix for HTTPS proxy. DEPRECATED, please use Proxy Configuration Service instead.")
+ .withDescription("Proxy host name or IP. DEPRECATED, please use Proxy Configuration Service instead.")
.build();
EXTENSIONAPI static constexpr auto ProxyPort = core::PropertyDefinitionBuilder<>::createProperty("Proxy Port")
.withDescription("The port number of the proxy host. DEPRECATED, please use Proxy Configuration Service instead.")
diff --git a/extensions/aws/tests/PutS3ObjectTests.cpp b/extensions/aws/tests/PutS3ObjectTests.cpp
index d868173f59..ca4b35b34c 100644
--- a/extensions/aws/tests/PutS3ObjectTests.cpp
+++ b/extensions/aws/tests/PutS3ObjectTests.cpp
@@ -19,6 +19,7 @@
#include "S3TestsFixture.h"
#include "processors/PutS3Object.h"
#include "unit/TestUtils.h"
+#include "catch2/generators/catch_generators.hpp"
namespace {
@@ -210,9 +211,21 @@ TEST_CASE_METHOD(PutS3ObjectTestsFixture, "Test proxy setting", "[awsS3Proxy]")
checkProxySettings();
}
+TEST_CASE_METHOD(PutS3ObjectTestsFixture, "HTTP scheme is stripped from the proxy host", "[awsS3Proxy]") {
+ const auto proxy_host = GENERATE("host", "http://host", "HTTP://host");
+ setRequiredProperties();
+ REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Host", proxy_host));
+ REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Port", "1234"));
+ test_controller.runSession(plan);
+ CHECK(mock_s3_request_sender_ptr->getClientConfig().proxyScheme == Aws::Http::Scheme::HTTP);
+ CHECK(mock_s3_request_sender_ptr->getClientConfig().proxyHost == "host");
+ CHECK(mock_s3_request_sender_ptr->getClientConfig().proxyPort == 1234);
+}
+
TEST_CASE_METHOD(PutS3ObjectTestsFixture, "HTTPS proxy is not supported", "[awsS3Proxy]") {
+ const auto proxy_host = GENERATE("https://host", "HTTPS://host");
setRequiredProperties();
- REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Host", "https://host"));
+ REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Host", proxy_host));
REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Port", "1234"));
REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Username", "username"));
REQUIRE(this->plan->setProperty(this->s3_processor, "Proxy Password", "password"));
diff --git a/extensions/aws/tests/S3TestsFixture.h b/extensions/aws/tests/S3TestsFixture.h
index 83e76012af..30c8bf34d7 100644
--- a/extensions/aws/tests/S3TestsFixture.h
+++ b/extensions/aws/tests/S3TestsFixture.h
@@ -144,7 +144,7 @@ class S3TestsFixture {
void checkProxySettings() {
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyScheme == Aws::Http::Scheme::HTTP);
- REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyHost == "http://host");
+ REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyHost == "host");
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyPort == 1234);
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyUserName == "username");
REQUIRE(mock_s3_request_sender_ptr->getClientConfig().proxyPassword == "password");
diff --git a/extensions/aws/tests/features/s3.feature b/extensions/aws/tests/features/s3.feature
index 4f9fd07057..ffdbfa52c2 100644
--- a/extensions/aws/tests/features/s3.feature
+++ b/extensions/aws/tests/features/s3.feature
@@ -38,7 +38,7 @@ Feature: Sending data from MiNiFi-C++ to an AWS server
And the object content type on the s3 server is "application/octet-stream" and the object metadata matches use metadata
And the Minifi logs contain the following message: "in a single upload" in less than 10 seconds
- Scenario Outline: A MiNiFi instance transfers encoded data through a http proxy to s3
+ Scenario: A MiNiFi instance transfers encoded data through a http proxy to s3
Given a GetFile processor with the "Input Directory" property set to "/tmp/input"
And a directory at "/tmp/input" has a file with the content "LH_O#L|FD bool: