diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java index 2718bc093..dcdfe1b49 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java @@ -49,6 +49,7 @@ import org.springframework.core.Ordered; import org.springframework.core.env.Profiles; import org.springframework.core.io.InputStreamResource; +import org.springframework.core.io.Resource; import org.springframework.util.AntPathMatcher; import org.springframework.util.ObjectUtils; import org.springframework.util.PathMatcher; @@ -160,94 +161,88 @@ public Environment findOne(String specifiedApplication, String specifiedProfiles private void addPropertySources(Environment environment, List apps, String[] profiles, List labels) { + String[] profilesByPrecedence = profilesByPrecedence(profiles); for (String label : labels) { - for (String profile : profiles) { + for (String profile : profilesByPrecedence) { addPropertySourcesForApps(apps, - app -> addProfileSpecificPropertySource(environment, app, profile, label)); + app -> addProfileSpecificPropertySource(environment, app, profile, profiles, label)); } - } - - // If we have no profiles just add property sources for all apps - if (profiles.length == 0) { - for (String label : labels) { - addPropertySourcesForApps(apps, - app -> addNonProfileSpecificPropertySource(environment, app, null, label)); - // Even with no profiles, negated profile documents (e.g. on-profile: - // "!my-profile") should be included because no profile is active, - // so all negations are satisfied + if (profiles.length == 0 && !this.searchPaths.isEmpty()) { addPropertySourcesForApps(apps, - app -> addNegatedProfilePropertySource(environment, app, profiles, label)); + app -> addProfileSpecificPropertySource(environment, app, null, profiles, label)); } } - else { + if (this.searchPaths.isEmpty()) { for (String label : labels) { - // If we have profiles, we still need to add property sources from files - // that - // are not profile specific but we pass - // along the profiles as well so we can check if any non-profile specific - // YAML - // files have profile specific documents - // within them - for (String profile : profiles) { - addPropertySourcesForApps(apps, - app -> addNonProfileSpecificPropertySource(environment, app, profile, label)); - } - // Handle documents with negated profile expressions (e.g. on-profile: - // "!my-profile") - // once per label rather than once per profile to avoid duplicates addPropertySourcesForApps(apps, - app -> addNegatedProfilePropertySource(environment, app, profiles, label)); + app -> addNonProfileSpecificPropertySource(environment, app, profiles, label)); } } } + /** + * Orders the profiles from the highest precedence to the lowest. As in the other + * environment repositories, the last profile wins. + * @param profiles the profiles in the order they were requested + * @return the profiles in reverse order + */ + private static String[] profilesByPrecedence(String[] profiles) { + List reversed = new ArrayList<>(Arrays.asList(profiles)); + Collections.reverse(reversed); + return reversed.toArray(new String[0]); + } + private void addPropertySourcesForApps(List apps, Consumer addPropertySource) { apps.forEach(addPropertySource); } - private void addNegatedProfilePropertySource(Environment environment, String app, String[] allProfiles, + private void addProfileSpecificPropertySource(Environment environment, String app, String profile, + String[] profiles, String label) { + List s3ConfigFiles = searchPaths.isEmpty() + ? getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, + (application, fileProfile, fileLabel) -> getYamlDocuments(null, application, fileProfile, + profiles, fileLabel)) + : getS3ConfigFileWithSearchPaths(app, profile, label, + key -> wrapKeyWithConfigFiles(key, app, profiles, label)); + addPropertySource(environment, s3ConfigFiles); + } + + private void addNonProfileSpecificPropertySource(Environment environment, String app, String[] profiles, String label) { - List s3ConfigFiles = this.searchPaths.isEmpty() - ? getNegatedProfileS3ConfigFileYaml(app, allProfiles, label) : getS3ConfigFileWithSearchPaths(app, null, - label, key -> wrapKeyWithNegatedConfigFiles(key, app, allProfiles, label)); + List s3ConfigFiles = new ArrayList<>(); + S3ConfigFile configFile = getS3PropertiesOrJsonConfigFile(app, null, label); + if (configFile != null) { + s3ConfigFiles.add(configFile); + } + s3ConfigFiles.addAll(getYamlDocuments(null, app, null, profiles, label)); addPropertySource(environment, s3ConfigFiles); } - private List getNegatedProfileS3ConfigFileYaml(String application, String[] allProfiles, + /** + * Returns the documents of a YAML file that apply to the profiles, from the highest + * precedence to the lowest. As in Spring Boot, a later document of a multi-document + * file takes precedence over an earlier one. + * @param key the key of the file, or {@code null} for the file of the application + * @param application the application + * @param profile the profile of a profile-specific file, or {@code null} + * @param profiles the requested profiles + * @param label the label + * @return the documents that apply to the profiles + */ + private List getYamlDocuments(String key, String application, String profile, String[] profiles, String label) { - List configFiles = new ArrayList<>(); try { - S3ConfigFile configFile = new NegatedProfileYamlDocumentS3ConfigFile(application, label, bucketName, - useApplicationAsDirectory, s3Client, allProfiles); - configFiles.add(configFile); - } - catch (IllegalStateException e) { - LOG.warn("Could not read YAML file using application <" + application + "> label <" + label + ">.", e); + return new MultiDocumentYamlS3ConfigFile(key, application, profile, label, bucketName, + useApplicationAsDirectory, s3Client) + .documentsFor(profiles); } catch (Exception e) { if (LOG.isDebugEnabled()) { - LOG.debug("Did not find negated profile yaml document in non-profile specific file using application <" - + application + "> label <" + label + ">."); + LOG.debug("Did not find yaml documents using application <" + application + "> profile <" + profile + + "> key <" + key + "> label <" + label + ">."); } + return Collections.emptyList(); } - return configFiles; - } - - private void addProfileSpecificPropertySource(Environment environment, String app, String profile, String label) { - List s3ConfigFiles = searchPaths.isEmpty() - ? getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile, - this::getProfileSpecificS3ConfigFileYaml) - : getS3ConfigFileWithSearchPaths(app, profile, label, - key -> wrapKeyWithConfigFiles(key, app, profile, label)); - addPropertySource(environment, s3ConfigFiles); - } - - private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile, - String label) { - List s3ConfigFiles = searchPaths.isEmpty() ? getS3ConfigFile(app, profile, label, - this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml) - : Collections.emptyList(); - addPropertySource(environment, s3ConfigFiles); } private void addPropertySource(Environment environment, List s3ConfigFiles) { @@ -257,19 +252,12 @@ private void addPropertySource(Environment environment, List s3Con } try { final Properties config = s3ConfigFile.read(); - // This logic handles the case where the s3 file is a YAML file that is - // not profile specific (ie it does not have - in the name) - // and does not have any profile specific documents in it. In this case we - // do - // not want to include this - // property source we only want to include the document for the default - // profile. When we create - // the S3ConfigFile for this file we set the - // shouldIncludeWithEmptyProperties to false - // in ProfileSpecificYamlDocumentS3ConfigFile for this specific case. - if (config == null || (config.isEmpty() && !s3ConfigFile.isShouldIncludeWithEmptyProperties())) { + if (config == null) { continue; } + // Apply the overrides before looking for the same source, which already + // has them + config.putAll(serverProperties.getOverrides()); String name = s3ConfigFile.getName(); boolean exists = environment.getPropertySources() .stream() @@ -278,7 +266,6 @@ private void addPropertySource(Environment environment, List s3Con continue; } environment.setVersion(s3ConfigFile.getVersion()); - config.putAll(serverProperties.getOverrides()); PropertySource propertySource = new PropertySource(name, config); if (LOG.isDebugEnabled()) { LOG.debug("Adding property source to environment " + propertySource); @@ -432,38 +419,14 @@ private boolean hasSupportedExtension(String key) { return key.endsWith(".properties") || key.endsWith(".json") || key.endsWith(".yml") || key.endsWith(".yaml"); } - private List wrapKeyWithConfigFiles(String key, String application, String profile, String label) { - + private List wrapKeyWithConfigFiles(String key, String application, String[] profiles, String label) { if (key.endsWith(".yml") || key.endsWith(".yaml")) { - List files = new ArrayList<>(); - files.addAll(getProfileSpecificYamlFromKey(key, application, profile, label)); - files.addAll(getNonProfileSpecificYamlFromKey(key, application, profile, label)); - return files; + return getYamlDocuments(key, application, null, profiles, label); } - return createConfigFileFromKey(key, application, profile, label).map(Collections::singletonList) + return createConfigFileFromKey(key, application, null, label).map(Collections::singletonList) .orElseGet(Collections::emptyList); } - private List getProfileSpecificYamlFromKey(String key, String application, String profile, - String label) { - - S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, profile, label, bucketName, - useApplicationAsDirectory, s3Client, - properties -> YamlS3ConfigFile.profileMatchesActivateProperty(profile, properties) - ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); - config.setShouldIncludeWithEmptyProperties(false); - return List.of(config); - } - - private List getNonProfileSpecificYamlFromKey(String key, String application, String profile, - String label) { - - S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, profile, label, bucketName, - useApplicationAsDirectory, s3Client, properties -> !YamlS3ConfigFile.onProfilePropertyExists(properties) - ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND); - return List.of(config); - } - private String extractPrefix(String pattern) { int firstWildcardIdx = -1; int starIdx = pattern.indexOf('*'); @@ -495,56 +458,6 @@ private Optional createConfigFileFromKey(String key, String applic return Optional.empty(); } - private List getNonProfileSpecificS3ConfigFileYaml(String application, String profile, - String label) { - List configFiles = new ArrayList<>(); - if (profile != null) { - try { - YamlS3ConfigFile configFileDocument = new ProfileSpecificYamlDocumentS3ConfigFile(application, profile, - label, bucketName, useApplicationAsDirectory, s3Client); - configFileDocument.setShouldIncludeWithEmptyProperties(false); - configFiles.add(configFileDocument); - } - catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug("Did not find specific yaml document in non-profile specific file using application <" - + application + "> profile <" + profile + "> label <" + label + ">."); - } - } - } - try { - YamlS3ConfigFile configFile = new NonProfileSpecificYamlDocumentS3ConfigFile(application, null, label, - bucketName, useApplicationAsDirectory, s3Client); - configFiles.add(configFile); - } - catch (Exception e) { - if (LOG.isDebugEnabled()) { - LOG.debug( - "Did not find non-profile specific yaml document in non-profile specific file using application <" - + application + "> profile " + " label <" + label + ">."); - } - } - return configFiles; - } - - private List getProfileSpecificS3ConfigFileYaml(String application, String profile, - String label) { - try { - YamlS3ConfigFile configFile = new ProfileSpecificYamlS3ConfigFile(application, profile, label, bucketName, - useApplicationAsDirectory, s3Client); - return List.of(configFile); - } - catch (Exception e) { - LOG.warn("Could not read YAML file", e); - return Collections.emptyList(); - } - } - - private S3ConfigFile getNonProfileSpecificPropertiesOrJsonConfigFile(String application, String profile, - String label) { - return getS3PropertiesOrJsonConfigFile(application, null, label); - } - private S3ConfigFile getS3PropertiesOrJsonConfigFile(String application, String profile, String label) { if (LOG.isDebugEnabled()) { LOG.debug("Getting S3 config file for " + application + " " + profile + " " + label); @@ -572,33 +485,6 @@ private S3ConfigFile getS3PropertiesOrJsonConfigFile(String application, String } } - private List wrapKeyWithNegatedConfigFiles(String key, String application, String[] allProfiles, - String label) { - if (key.endsWith(".yml") || key.endsWith(".yaml")) { - S3ConfigFileFromKey config = new S3ConfigFileFromKey(key, application, null, label, bucketName, - this.useApplicationAsDirectory, s3Client, properties -> { - Object onProfileValue = properties.get("spring.config.activate.on-profile"); - if (onProfileValue == null) { - onProfileValue = properties.get("spring.config.activate.onProfile"); - } - if (onProfileValue == null) { - return YamlProcessor.MatchStatus.NOT_FOUND; - } - - String expression = onProfileValue.toString().trim(); - if (AwsS3EnvironmentRepository.isSimpleProfileName(expression)) { - return YamlProcessor.MatchStatus.NOT_FOUND; - } - List allProfilesList = Arrays.asList(allProfiles); - boolean matches = Profiles.of(expression).matches(allProfilesList::contains); - return matches ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND; - }); - config.setShouldIncludeWithEmptyProperties(false); - return List.of(config); - } - return Collections.emptyList(); - } - @Override public Locations getLocations(String application, String profiles, String label) { StringBuilder baseLocation = new StringBuilder(AWS_S3_RESOURCE_SCHEME + bucketName + PATH_SEPARATOR); @@ -620,7 +506,7 @@ static boolean isSimpleProfileName(String expression) { interface YamlS3ConfigFileCreator { - List create(String application, String profile, String label); + List create(String application, String profile, String label); } @@ -650,8 +536,6 @@ abstract class S3ConfigFile { private String version; - private boolean shouldIncludeWithEmptyProperties = true; - private final boolean useApplicationAsDirectory; protected S3ConfigFile(String application, String profile, String label, String bucketName, @@ -668,15 +552,11 @@ String getVersion() { return version; } - abstract Properties read(); - - boolean isShouldIncludeWithEmptyProperties() { - return shouldIncludeWithEmptyProperties; + void setVersion(String version) { + this.version = version; } - void setShouldIncludeWithEmptyProperties(boolean shouldIncludeWithEmptyProperties) { - this.shouldIncludeWithEmptyProperties = shouldIncludeWithEmptyProperties; - } + abstract Properties read(); public String getName() { return createPropertySourceName(application, profile); @@ -731,7 +611,7 @@ String buildObjectKeyPrefix(boolean includeProfile) { return objectKeyPrefix.toString(); } - private String createPropertySourceName(String app, String profile) { + String createPropertySourceName(String app, String profile) { StringBuilder propertySourceName = new StringBuilder().append("s3:").append(app); if (this.useApplicationAsDirectory) { propertySourceName.append(PATH_SEPARATOR).append("application"); @@ -777,35 +657,12 @@ protected List getExtensions() { class YamlS3ConfigFile extends S3ConfigFile { - final YamlProcessor.DocumentMatcher[] documentMatchers; - YamlS3ConfigFile(String application, String profile, String label, String bucketName, boolean useApplicationAsDirectory, S3Client s3Client) { - this(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, - new YamlProcessor.DocumentMatcher[] {}); - } - - YamlS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, - final YamlProcessor.DocumentMatcher... documentMatchers) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); - this.documentMatchers = documentMatchers; this.properties = read(); } - protected static boolean profileMatchesActivateProperty(String profile, Properties properties) { - if (profile == null) { - return false; - } - return profile.equals(properties.get("spring.config.activate.on-profile")) - || profile.equals(properties.get("spring.config.activate.onProfile")); - } - - protected static boolean onProfilePropertyExists(Properties properties) { - return properties.get("spring.config.activate.on-profile") != null - || properties.get("spring.config.activate.onProfile") != null; - } - @Override public Properties read() { if (properties != null) { @@ -814,7 +671,6 @@ public Properties read() { final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); try (InputStream in = getObject()) { yaml.setResources(new InputStreamResource(in)); - yaml.setDocumentMatchers(documentMatchers); return yaml.getObject(); } catch (Exception e) { @@ -830,55 +686,6 @@ protected List getExtensions() { } -class ProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile { - - ProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, - properties -> profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND - : YamlProcessor.MatchStatus.NOT_FOUND); - } - - @Override - public boolean isShouldIncludeWithEmptyProperties() { - return false; - } - - @Override - protected String buildObjectKeyPrefix() { - return super.buildObjectKeyPrefix(false); - } - -} - -class NonProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile { - - NonProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, - properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.FOUND - : YamlProcessor.MatchStatus.NOT_FOUND); - } - - @Override - public boolean isShouldIncludeWithEmptyProperties() { - return false; - } - -} - -class ProfileSpecificYamlS3ConfigFile extends YamlS3ConfigFile { - - ProfileSpecificYamlS3ConfigFile(String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client) { - super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client, - properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.ABSTAIN - : profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND - : YamlProcessor.MatchStatus.NOT_FOUND); - } - -} - class JsonS3ConfigFile extends YamlS3ConfigFile { // YAML is a superset of JSON, which means you can parse JSON with a YAML parser @@ -900,18 +707,10 @@ class S3ConfigFileFromKey extends S3ConfigFile { private final String key; - private final YamlProcessor.DocumentMatcher[] documentMatchers; - - S3ConfigFileFromKey(String key, String application, String profile, String label, String bucketName, - S3Client s3Client) { - this(key, application, profile, label, bucketName, false, s3Client, new YamlProcessor.DocumentMatcher[] {}); - } - S3ConfigFileFromKey(String key, String application, String profile, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, YamlProcessor.DocumentMatcher... documentMatchers) { + boolean useApplicationAsDirectory, S3Client s3Client) { super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); this.key = key; - this.documentMatchers = documentMatchers; this.properties = read(); } @@ -947,15 +746,16 @@ public Properties read() { } return props; } - else if ("json".equals(ext) || "yml".equals(ext) || "yaml".equals(ext)) { + else if ("json".equals(ext)) { + // YAML is a superset of JSON, which means you can parse JSON with a YAML + // parser final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); try (InputStream in = getObject()) { yaml.setResources(new InputStreamResource(in)); - yaml.setDocumentMatchers(documentMatchers); return yaml.getObject(); } catch (Exception e) { - LOG.warn("Could not read YAML/JSON file", e); + LOG.warn("Could not read JSON file", e); throw new IllegalStateException("Cannot load environment", e); } } @@ -964,53 +764,191 @@ else if ("json".equals(ext) || "yml".equals(ext) || "yaml".equals(ext)) { } -class NegatedProfileYamlDocumentS3ConfigFile extends YamlS3ConfigFile { +/** + * A YAML file of possibly several documents. As in Spring Boot, a later document takes + * precedence over an earlier one, and a document applies to the requested profiles when + * it has no {@code spring.config.activate.on-profile} property or when one of its profile + * expressions matches them. + */ +class MultiDocumentYamlS3ConfigFile extends S3ConfigFile { + + private static final List ACTIVATION_PROPERTIES = List.of("spring.config.activate.on-profile", + "spring.config.activate.onProfile"); + + private final String key; + + private final List documents; + + MultiDocumentYamlS3ConfigFile(String key, String application, String profile, String label, String bucketName, + boolean useApplicationAsDirectory, S3Client s3Client) { + super(application, profile, label, bucketName, useApplicationAsDirectory, s3Client); + this.key = key; + this.documents = readDocuments(); + } + + /** + * Returns a config file for each non-empty document that applies to the profiles, + * from the highest precedence to the lowest. Documents that would get the same name + * are told apart by their position in the file. + * @param profiles the requested profiles + * @return the config files of the documents + */ + List documentsFor(String[] profiles) { + List applying = new ArrayList<>(); + for (int index = this.documents.size() - 1; index >= 0; index--) { + Properties document = this.documents.get(index); + if (!document.isEmpty() && appliesTo(document, index, profiles)) { + applying.add(new Document(index, document, nameOf(document, profiles))); + } + } + List names = applying.stream().map(Document::name).toList(); + return applying.stream().map(document -> { + String name = (Collections.frequency(names, document.name()) > 1) + ? document.name() + " (document #" + (document.index() + 1) + ")" : document.name(); + return (S3ConfigFile) new YamlDocumentS3ConfigFile(name, document.properties(), getVersion()); + }).toList(); + } + + private boolean appliesTo(Properties document, int index, String[] profiles) { + List expressions = activationExpressions(document); + if (expressions.isEmpty()) { + return true; + } + try { + List activeProfiles = Arrays.asList(profiles); + return Profiles.of(expressions.toArray(new String[0])).matches(activeProfiles::contains); + } + catch (IllegalArgumentException e) { + LOG.warn("Skipping document #" + (index + 1) + " of " + getName() + " with the invalid profile expression " + + expressions + ": " + e.getMessage()); + return false; + } + } - private final String[] allProfiles; + private String nameOf(Properties document, String[] profiles) { + if (this.key != null) { + return "s3:" + this.bucketName + "/" + this.key; + } + if (this.profile != null) { + return createPropertySourceName(this.application, this.profile); + } + List expressions = activationExpressions(document); + if (expressions.isEmpty()) { + return createPropertySourceName(this.application, null); + } + if (expressions.size() == 1 && AwsS3EnvironmentRepository.isSimpleProfileName(expressions.get(0))) { + return createPropertySourceName(this.application, expressions.get(0)); + } + // A document matched by a complex or negated expression is named after the + // requested profiles, so that its name never collides with the one of the + // document that applies to every profile + String suffix = String.join(",", profiles); + return suffix.isEmpty() ? createPropertySourceName(this.application, null) + : createPropertySourceName(this.application, suffix); + } - NegatedProfileYamlDocumentS3ConfigFile(String application, String label, String bucketName, - boolean useApplicationAsDirectory, S3Client s3Client, String[] allProfiles) { - super(application, null, label, bucketName, useApplicationAsDirectory, s3Client, properties -> { - Object onProfileValue = properties.get("spring.config.activate.on-profile"); - if (onProfileValue == null) { - onProfileValue = properties.get("spring.config.activate.onProfile"); + /** + * Returns the profile expressions of a document: a comma-separated list, or the + * elements of a YAML list. + */ + private static List activationExpressions(Properties document) { + for (String activationProperty : ACTIVATION_PROPERTIES) { + Object value = document.get(activationProperty); + if (value != null) { + return Arrays.stream(StringUtils.commaDelimitedListToStringArray(value.toString())) + .map(String::trim) + .toList(); } - if (onProfileValue == null) { - return YamlProcessor.MatchStatus.NOT_FOUND; + List elements = new ArrayList<>(); + for (int i = 0; document.get(activationProperty + "[" + i + "]") != null; i++) { + elements.add(document.get(activationProperty + "[" + i + "]").toString().trim()); } - String expression = onProfileValue.toString().trim(); - // Simple positive profile names are already handled by - // ProfileSpecificYamlDocumentS3ConfigFile. Only process complex or negated - // expressions here to avoid adding duplicate property sources. - if (AwsS3EnvironmentRepository.isSimpleProfileName(expression)) { - return YamlProcessor.MatchStatus.NOT_FOUND; + if (!elements.isEmpty()) { + return elements; } - List allProfilesList = Arrays.asList(allProfiles); - boolean matches = Profiles.of(expression).matches(allProfilesList::contains); - return matches ? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND; - }); - this.allProfiles = allProfiles; + } + return Collections.emptyList(); + } + + private List readDocuments() { + try (InputStream in = getObject()) { + return new DocumentsLoader(new InputStreamResource(in)).load(); + } + catch (Exception e) { + LOG.warn("Could not read YAML file", e); + throw new IllegalStateException("Cannot load environment", e); + } + } + + /** + * Returns the properties of all the documents, a later document overriding an earlier + * one. The documents are added with {@link #documentsFor(String[])}. + * @return the properties of the file + */ + @Override + Properties read() { + Properties properties = new Properties(); + this.documents.forEach(properties::putAll); + return properties; } @Override protected String buildObjectKeyPrefix() { - return super.buildObjectKeyPrefix(false); + return (this.key != null) ? this.key.substring(0, this.key.lastIndexOf('.')) : super.buildObjectKeyPrefix(); } @Override - public boolean isShouldIncludeWithEmptyProperties() { - return false; + protected List getExtensions() { + return (this.key != null) ? List.of(this.key.substring(this.key.lastIndexOf('.') + 1)) : List.of("yml", "yaml"); + } + + private record Document(int index, Properties properties, String name) { + + } + + private static final class DocumentsLoader extends YamlProcessor { + + DocumentsLoader(Resource resource) { + setResources(resource); + } + + List load() { + List documents = new ArrayList<>(); + process((properties, map) -> documents.add(properties)); + return documents; + } + + } + +} + +/** + * A document of a YAML file. + */ +class YamlDocumentS3ConfigFile extends S3ConfigFile { + + private final String name; + + YamlDocumentS3ConfigFile(String name, Properties properties, String version) { + super(null, null, null, null, false, null); + this.name = name; + this.properties = properties; + setVersion(version); } @Override public String getName() { - // This document is matched by evaluating a complex/negated on-profile - // expression rather than an exact profile name, so it isn't looked up by a - // profile-suffixed key. Suffix the name with the active profiles anyway so it - // never collides with the plain, non-profile-specific source's name (which - // would otherwise cause one to silently replace the other on the client). - String suffix = String.join(",", allProfiles); - return suffix.isEmpty() ? super.getName() : super.getName() + "-" + suffix; + return this.name; + } + + @Override + Properties read() { + return this.properties; + } + + @Override + protected List getExtensions() { + return Collections.emptyList(); } } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java index f7478afb0..170802be8 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java @@ -288,18 +288,14 @@ public void twoNegatedProfileDocumentsBothIncludedWhenNeitherProfileIsActive() t List propertySources = env.getPropertySources(); - // Both negated profile documents match and are merged into one property source - Optional negatedSource = propertySources.stream() - .filter(ps -> ps.getSource().containsKey("demo.negatedProfileMarker") - || ps.getSource().containsKey("demo.otherNegatedMarker")) - .findFirst(); - assertThat(negatedSource) + // Both negated profile documents match, each in its own property source + assertThat(propertySources) .as("negated profile documents should be present when neither 'my-profile' nor 'other-profile' is active") - .isPresent(); - assertThat(negatedSource.get().getSource().get("demo.negatedProfileMarker")) - .isEqualTo("present-when-my-profile-is-not-active"); - assertThat(negatedSource.get().getSource().get("demo.otherNegatedMarker")) - .isEqualTo("present-when-other-profile-is-not-active"); + .anySatisfy(ps -> assertThat(ps.getSource().get("demo.negatedProfileMarker")) + .isEqualTo("present-when-my-profile-is-not-active")) + .anySatisfy(ps -> assertThat(ps.getSource().get("demo.otherNegatedMarker")) + .isEqualTo("present-when-other-profile-is-not-active")); + assertThat(propertySources).extracting(PropertySource::getName).doesNotHaveDuplicates(); Optional baseSource = propertySources.stream() .filter(ps -> ps.getSource().containsKey("demo.base")) @@ -544,14 +540,149 @@ public void findWithDefaultProfileUsingSuffix() { @Test public void findWithMultipleProfilesAllFound() { - putFiles("foo-profile1.yml", yamlContent); - String versionId = putFiles("foo-profile2.yml", jsonContent); + // the version is the one of the last property source, here the one of the + // first profile, which has the lowest precedence + String versionId = putFiles("foo-profile1.yml", yamlContent); + putFiles("foo-profile2.yml", jsonContent); final Environment env = envRepo.findOne("foo", "profile1,profile2", null); assertExpectedEnvironment(env, "foo", null, versionId, 2, "profile1", "profile2"); } + @Test + public void lastProfileTakesPrecedenceForProfileSpecificFiles() { + putFiles("foo-profile1.properties", "greeting=profile1"); + putFiles("foo-profile2.properties", "greeting=profile2"); + + final Environment env = envRepo.findOne("foo", "profile1,profile2", null); + + assertThat(env.getProfiles()).containsExactly("profile1", "profile2"); + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("profile2", "profile1"); + } + + @Test + public void applicationSpecificFileTakesPrecedenceOverProfileDocumentOfSharedFile() { + putFiles("foo.yml", "greeting: foo\n"); + putFiles("application.yml", + "greeting: application\n---\nspring.config.activate.on-profile: profile1\ngreeting: shared\n"); + + final Environment env = envRepo.findOne("foo", "profile1", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("foo", "shared", "application"); + } + + @Test + public void laterDocumentTakesPrecedenceInAMultiDocumentFile() { + putFiles("foo.yml", "greeting: first\n---\nspring.config.activate.on-profile: profile1\ngreeting: profile1\n" + + "---\ngreeting: last\n"); + + final Environment env = envRepo.findOne("foo", "profile1", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("last", "profile1", "first"); + assertThat(env.getPropertySources()).extracting(PropertySource::getName).doesNotHaveDuplicates(); + } + + @Test + public void documentOrderDecidesBetweenTheProfileDocumentsOfAFile() { + putFiles("foo.yml", "greeting: default\n---\nspring.config.activate.on-profile: profile2\ngreeting: profile2\n" + + "---\nspring.config.activate.on-profile: profile1\ngreeting: profile1\n"); + + final Environment env = envRepo.findOne("foo", "profile1,profile2", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("profile1", "profile2", "default"); + } + + @Test + public void negatedProfileDocumentFollowsTheDocumentOrder() { + putFiles("foo.yml", + "greeting: default\n---\nspring.config.activate.on-profile: \"!other\"\ngreeting: not-other\n"); + + final Environment env = envRepo.findOne("foo", "profile1", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("not-other", "default"); + } + + @Test + public void searchPathFileFollowsTheDocumentOrder() { + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, + List.of("{label}/{application}.yml")); + putFiles("v1/app.yml", "greeting: first\n---\nspring.config.activate.on-profile: profile1\ngreeting: profile1\n" + + "---\ngreeting: last\n"); + + Environment env = repo.findOne("app", "profile1", "v1"); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("last", "profile1", "first"); + assertThat(env.getPropertySources()).extracting(PropertySource::getName).doesNotHaveDuplicates(); + } + + @Test + public void profileSpecificFileHonoursTheProfileExpressionsOfItsDocuments() { + putFiles("foo-profile1.yml", + "greeting: plain\n---\nspring.config.activate.on-profile: \"!other\"\ngreeting: not-other\n"); + + final Environment env = envRepo.findOne("foo", "profile1", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("not-other", "plain"); + assertThat(env.getPropertySources()).extracting(PropertySource::getName).doesNotHaveDuplicates(); + } + + @Test + public void invalidProfileExpressionSkipsOnlyItsDocument() { + putFiles("foo.yml", + "greeting: plain\n---\nspring.config.activate.on-profile: \"profile1 & (other\"\ngreeting: invalid\n"); + + final Environment env = envRepo.findOne("foo", "profile1", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("plain"); + } + + @Test + public void commaSeparatedProfilesActivateADocument() { + putFiles("foo.yml", + "greeting: plain\n---\nspring.config.activate.on-profile: other,profile1\ngreeting: listed\n"); + + final Environment env = envRepo.findOne("foo", "profile1", null); + + assertThat(env.getPropertySources()).extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("listed", "plain"); + } + + @Test + public void listOfProfilesActivatesADocumentOnlyForThoseProfiles() { + putFiles("foo.yml", "greeting: plain\n---\nspring:\n config:\n activate:\n on-profile:\n" + + " - other\n - profile1\ngreeting: listed\n"); + + assertThat(envRepo.findOne("foo", "profile1", null).getPropertySources()) + .extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("listed", "plain"); + assertThat(envRepo.findOne("foo", "profile2", null).getPropertySources()) + .extracting(source -> (Object) source.getSource().get("greeting")) + .containsExactly("plain"); + } + + @Test + public void searchPathFileIsAddedOnceForSeveralProfilesWithOverrides() { + ConfigServerProperties serverWithOverrides = new ConfigServerProperties(); + serverWithOverrides.setOverrides(Map.of("overridden", "yes")); + AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, + serverWithOverrides, List.of("{label}/{application}.yml")); + putFiles("v1/app.yml", "greeting: plain\n"); + + Environment env = repo.findOne("app", "profile1,profile2", "v1"); + + assertThat(env.getPropertySources()).hasSize(1); + assertThat(env.getPropertySources().get(0).getSource().get("overridden")).isEqualTo("yes"); + } + @Test public void findWithMultipleProfilesOneFound() { String versionId = putFiles("foo-profile2.yml", jsonContent); @@ -685,8 +816,10 @@ public void findWithDefaultProfileUsingSuffix_ApplicationDirVariant() { @Test public void findWithMultipleProfilesAllFound_ApplicationDirVariant() { - putFiles("foo/application-profile1.yml", yamlContent); - String versionId = putFiles("foo/application-profile2.yml", jsonContent); + // the version is the one of the last property source, here the one of the + // first profile, which has the lowest precedence + String versionId = putFiles("foo/application-profile1.yml", yamlContent); + putFiles("foo/application-profile2.yml", jsonContent); final Environment env = envRepoApplicationDir.findOne("foo", "profile1,profile2", null); @@ -879,7 +1012,7 @@ public void searchPaths_applicationAsDirectory_shouldStillHonorSearchPaths() { } @Test - public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws IOException { + public void multiDocumentYaml_withSearchPaths_shouldKeepEveryDocumentUnderAUniqueName() throws IOException { List paths = List.of("{label}/{application}.yml"); AwsS3EnvironmentRepository repo = new AwsS3EnvironmentRepository(s3Client, "bucket1", false, server, paths); @@ -887,10 +1020,10 @@ public void multiDocumentYaml_withSearchPaths_shouldNotSplitDocuments() throws I putFiles("lab/app.yml", multi); Environment env = repo.findOne("app", "", "lab"); - assertThat(env.getPropertySources()).hasSize(1); - @SuppressWarnings("unchecked") - Map map = (Map) env.getPropertySources().get(0).getSource(); - assertThat(map).containsEntry("a", 1).containsEntry("b", 2); + assertThat(env.getPropertySources()).hasSize(2); + assertThat(env.getPropertySources()).extracting(PropertySource::getName).doesNotHaveDuplicates(); + assertThat(env.getPropertySources().get(0).getSource().get("b")).isEqualTo(2); + assertThat(env.getPropertySources().get(1).getSource().get("a")).isEqualTo(1); } @Test