diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a975f9a6ba..588dd6d922f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Breaking Changes ### Additions and Improvements +- Add support to load external profiles using `--profile` [#7265](https://github.com/hyperledger/besu/issues/7265) ### Bug fixes diff --git a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java index 8b123e70d36..427cac92480 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/BesuCommand.java @@ -39,7 +39,7 @@ import org.hyperledger.besu.chainimport.RlpBlockImporter; import org.hyperledger.besu.cli.config.EthNetworkConfig; import org.hyperledger.besu.cli.config.NetworkName; -import org.hyperledger.besu.cli.config.ProfileName; +import org.hyperledger.besu.cli.config.ProfilesCompletionCandidates; import org.hyperledger.besu.cli.converter.MetricCategoryConverter; import org.hyperledger.besu.cli.converter.PercentageConverter; import org.hyperledger.besu.cli.converter.SubnetInfoConverter; @@ -565,9 +565,10 @@ private InetAddress autoDiscoverDefaultIP() { @Option( names = {PROFILE_OPTION_NAME}, paramLabel = PROFILE_FORMAT_HELP, + completionCandidates = ProfilesCompletionCandidates.class, description = "Overwrite default settings. Possible values are ${COMPLETION-CANDIDATES}. (default: none)") - private final ProfileName profile = null; + private String profile = null; // don't set it as final due to picocli completion candidates @Option( names = {"--nat-method"}, @@ -2773,7 +2774,7 @@ private String generateConfigurationOverview() { } if (profile != null) { - builder.setProfile(profile.toString()); + builder.setProfile(profile); } builder.setHasCustomGenesis(genesisFile != null); diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java b/besu/src/main/java/org/hyperledger/besu/cli/config/InternalProfileName.java similarity index 59% rename from besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java rename to besu/src/main/java/org/hyperledger/besu/cli/config/InternalProfileName.java index 8c17dfb4f2c..efac7a52032 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfileName.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/InternalProfileName.java @@ -14,10 +14,18 @@ */ package org.hyperledger.besu.cli.config; +import java.util.Arrays; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + import org.apache.commons.lang3.StringUtils; -/** Enum for profile names. Each profile corresponds to a configuration file. */ -public enum ProfileName { +/** + * Enum for profile names which are bundled. Each profile corresponds to a bundled configuration + * file. + */ +public enum InternalProfileName { /** The 'STAKER' profile */ STAKER("profiles/staker.toml"), /** The 'MINIMALIST_STAKER' profile */ @@ -31,12 +39,36 @@ public enum ProfileName { private final String configFile; + /** + * Returns the InternalProfileName that matches the given name, ignoring case. + * + * @param name The profile name + * @return Optional InternalProfileName if found, otherwise empty + */ + public static Optional valueOfIgnoreCase(final String name) { + return Arrays.stream(values()) + .filter(profile -> profile.name().equalsIgnoreCase(name)) + .findFirst(); + } + + /** + * Returns the set of internal profile names as lowercase. + * + * @return Set of internal profile names + */ + public static Set getInternalProfileNames() { + return Arrays.stream(InternalProfileName.values()) + .map(InternalProfileName::name) + .map(String::toLowerCase) + .collect(Collectors.toSet()); + } + /** * Constructs a new ProfileName. * * @param configFile the configuration file corresponding to the profile */ - ProfileName(final String configFile) { + InternalProfileName(final String configFile) { this.configFile = configFile; } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidates.java b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidates.java new file mode 100644 index 00000000000..51e9c49a0ab --- /dev/null +++ b/besu/src/main/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidates.java @@ -0,0 +1,37 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.config; + +import org.hyperledger.besu.cli.util.ProfileFinder; + +import java.util.Iterator; +import java.util.Set; +import java.util.TreeSet; + +/** Provides a list of profile names that can be used for command line completion. */ +public class ProfilesCompletionCandidates implements Iterable { + /** + * Create a new instance of ProfilesCompletionCandidates. This constructor is required for + * Picocli. + */ + public ProfilesCompletionCandidates() {} + + @Override + public Iterator iterator() { + final Set profileNames = new TreeSet<>(InternalProfileName.getInternalProfileNames()); + profileNames.addAll(ProfileFinder.getExternalProfileNames()); + return profileNames.iterator(); + } +} diff --git a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java index f17f1a09fc6..12500748ce7 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/subcommands/ValidateConfigSubCommand.java @@ -70,7 +70,7 @@ public void run() { checkNotNull(parentCommand); try { TomlConfigurationDefaultProvider.fromFile(commandLine, dataPath.toFile()) - .loadConfigurationFromFile(); + .loadConfigurationIfNotLoaded(); } catch (Exception e) { this.out.println(e); return; diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java b/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java index edb94f667af..ba35f6b69db 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/ProfileFinder.java @@ -16,11 +16,19 @@ import static org.hyperledger.besu.cli.DefaultCommandValues.PROFILE_OPTION_NAME; -import org.hyperledger.besu.cli.config.ProfileName; +import org.hyperledger.besu.cli.config.InternalProfileName; +import java.io.IOException; import java.io.InputStream; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; +import java.util.stream.Stream; import picocli.CommandLine; @@ -50,30 +58,94 @@ protected String getConfigEnvName() { @Override public Optional getFromOption( final CommandLine.ParseResult parseResult, final CommandLine commandLine) { + final String profileName; try { - return getProfile(parseResult.matchedOption(PROFILE_OPTION_NAME).getter().get(), commandLine); - } catch (Exception e) { - throw new RuntimeException(e); + profileName = parseResult.matchedOption(PROFILE_OPTION_NAME).getter().get(); + } catch (final Exception e) { + throw new CommandLine.ParameterException( + commandLine, "Unexpected error in obtaining value of --profile", e); } + return getProfile(profileName, commandLine); } @Override public Optional getFromEnvironment( final Map environment, final CommandLine commandLine) { - return getProfile(ProfileName.valueOf(environment.get(PROFILE_ENV_NAME)), commandLine); + return getProfile(environment.get(PROFILE_ENV_NAME), commandLine); } private static Optional getProfile( - final ProfileName profileName, final CommandLine commandLine) { - return Optional.of(getTomlFile(commandLine, profileName.getConfigFile())); + final String profileName, final CommandLine commandLine) { + final Optional internalProfileConfigPath = + InternalProfileName.valueOfIgnoreCase(profileName).map(InternalProfileName::getConfigFile); + if (internalProfileConfigPath.isPresent()) { + return Optional.of(getTomlFileFromClasspath(internalProfileConfigPath.get())); + } else { + final Path externalProfileFile = defaultProfilesDir().resolve(profileName + ".toml"); + if (Files.exists(externalProfileFile)) { + try { + return Optional.of(Files.newInputStream(externalProfileFile)); + } catch (IOException e) { + throw new CommandLine.ParameterException( + commandLine, "Error reading external profile: " + profileName); + } + } else { + throw new CommandLine.ParameterException( + commandLine, "Unable to load external profile: " + profileName); + } + } } - private static InputStream getTomlFile(final CommandLine commandLine, final String file) { - InputStream resourceUrl = ProfileFinder.class.getClassLoader().getResourceAsStream(file); + private static InputStream getTomlFileFromClasspath(final String profileConfigFile) { + InputStream resourceUrl = + ProfileFinder.class.getClassLoader().getResourceAsStream(profileConfigFile); + // this is not meant to happen, because for each InternalProfileName there is a corresponding + // TOML file in resources if (resourceUrl == null) { - throw new CommandLine.ParameterException( - commandLine, String.format("TOML file %s not found", file)); + throw new IllegalStateException( + String.format("Internal Profile TOML %s not found", profileConfigFile)); } return resourceUrl; } + + /** + * Returns the external profile names which are file names without extension in the default + * profiles directory. + * + * @return Set of external profile names + */ + public static Set getExternalProfileNames() { + final Path profilesDir = defaultProfilesDir(); + if (!Files.exists(profilesDir)) { + return Set.of(); + } + + try (Stream pathStream = Files.list(profilesDir)) { + return pathStream + .filter(Files::isRegularFile) + .filter(path -> path.toString().endsWith(".toml")) + .map( + path -> + path.getFileName() + .toString() + .substring(0, path.getFileName().toString().length() - 5)) + .collect(Collectors.toSet()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + /** + * Return default profiles directory location + * + * @return Path to default profiles directory + */ + private static Path defaultProfilesDir() { + final String profilesDir = System.getProperty("besu.profiles.dir"); + if (profilesDir == null) { + return Paths.get(System.getProperty("besu.home", "."), "profiles"); + } else { + return Paths.get(profilesDir); + } + } } diff --git a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java index 1747f461dc4..ca76b40b686 100644 --- a/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java +++ b/besu/src/main/java/org/hyperledger/besu/cli/util/TomlConfigurationDefaultProvider.java @@ -96,7 +96,7 @@ public static TomlConfigurationDefaultProvider fromInputStream( @Override public String defaultValue(final ArgSpec argSpec) { - loadConfigurationFromFile(); + loadConfigurationIfNotLoaded(); // only options can be used in config because a name is needed for the key // so we skip default for positional params @@ -227,10 +227,10 @@ private String getNumericEntryAsString(final OptionSpec spec) { } private void checkConfigurationValidity() { - if (result == null || result.isEmpty()) + if (result == null || result.isEmpty()) { throw new ParameterException( - commandLine, - String.format("Unable to read TOML configuration file %s", configurationInputStream)); + commandLine, "Unable to read from empty TOML configuration file."); + } if (!isUnknownOptionsChecked && !commandLine.isUnmatchedArgumentsAllowed()) { checkUnknownOptions(result); @@ -239,8 +239,7 @@ private void checkConfigurationValidity() { } /** Load configuration from file. */ - public void loadConfigurationFromFile() { - + public void loadConfigurationIfNotLoaded() { if (result == null) { try { final TomlParseResult result = Toml.parse(configurationInputStream); @@ -289,12 +288,12 @@ private void checkUnknownOptions(final TomlParseResult result) { .collect(Collectors.toSet()); if (!unknownOptionsList.isEmpty()) { - final String options = unknownOptionsList.size() > 1 ? "options" : "option"; - final String csvUnknownOptions = - unknownOptionsList.stream().collect(Collectors.joining(", ")); + final String csvUnknownOptions = String.join(", ", unknownOptionsList); throw new ParameterException( commandLine, - String.format("Unknown %s in TOML configuration file: %s", options, csvUnknownOptions)); + String.format( + "Unknown option%s in TOML configuration file: %s", + unknownOptionsList.size() > 1 ? "s" : "", csvUnknownOptions)); } } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java index 641e69dba64..c6ffbdc1759 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/BesuCommandTest.java @@ -259,7 +259,7 @@ public void callingWithConfigOptionButNonExistingFileShouldDisplayHelp() throws final Path tempConfigFilePath = createTempFile("an-invalid-file-name-without-extension", ""); parseCommand("--config-file", tempConfigFilePath.toString()); - final String expectedOutputStart = "Unable to read TOML configuration file"; + final String expectedOutputStart = "Unable to read from empty TOML configuration file."; assertThat(commandErrorOutput.toString(UTF_8)).startsWith(expectedOutputStart); assertThat(commandOutput.toString(UTF_8)).isEmpty(); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java index 07e440430a5..841c6680007 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/ConfigurationOverviewBuilderTest.java @@ -20,7 +20,7 @@ import static org.hyperledger.besu.ethereum.eth.transactions.TransactionPoolConfiguration.Implementation.SEQUENCED; import static org.mockito.Mockito.mock; -import org.hyperledger.besu.cli.config.ProfileName; +import org.hyperledger.besu.cli.config.InternalProfileName; import org.hyperledger.besu.evm.internal.EvmConfiguration; import java.math.BigInteger; @@ -213,7 +213,7 @@ void setWorldStateUpdateModeJournaled() { @Test void setProfile() { - builder.setProfile(ProfileName.DEV.name()); + builder.setProfile(InternalProfileName.DEV.name()); final String profileSelected = builder.build(); assertThat(profileSelected).contains("Profile: DEV"); } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/ProfilesTest.java b/besu/src/test/java/org/hyperledger/besu/cli/ProfilesTest.java index cdfe09964ff..019ff7a7ed2 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/ProfilesTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/ProfilesTest.java @@ -17,21 +17,104 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.assertj.core.api.Assertions.assertThat; -import org.hyperledger.besu.cli.config.ProfileName; +import org.hyperledger.besu.cli.config.InternalProfileName; +import org.hyperledger.besu.cli.util.ProfileFinder; +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; import org.junit.jupiter.params.ParameterizedTest; -import org.junit.jupiter.params.provider.EnumSource; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; public class ProfilesTest extends CommandTestAbstract { + @TempDir private static Path tempProfilesDir; + private static String originalProfilesDirProperty; - /** Test if besu will validate the combination of options within the given profile. */ - @ParameterizedTest - @EnumSource(ProfileName.class) - public void testProfileWithNoOverrides_doesNotError(final ProfileName profileName) { + @BeforeAll + public static void copyExternalProfiles() throws IOException { + for (String internalProfileName : InternalProfileName.getInternalProfileNames()) { + final Path profilePath = tempProfilesDir.resolve(internalProfileName + "_external.toml"); + + String profileConfigFile = + InternalProfileName.valueOfIgnoreCase(internalProfileName).get().getConfigFile(); + try (InputStream resourceUrl = + ProfileFinder.class.getClassLoader().getResourceAsStream(profileConfigFile)) { + if (resourceUrl != null) { + Files.copy(resourceUrl, profilePath); + } + } + } + + // add an empty external profile + Files.createFile(tempProfilesDir.resolve("empty_external.toml")); + } - parseCommand("--profile", profileName.name()); + @BeforeAll + public static void setupSystemProperty() { + originalProfilesDirProperty = System.getProperty("besu.profiles.dir"); + // sets the system property for the test + System.setProperty("besu.profiles.dir", tempProfilesDir.toString()); + } + + static Stream profileNameProvider() { + final Set profileNames = new TreeSet<>(InternalProfileName.getInternalProfileNames()); + final Set externalProfileNames = + InternalProfileName.getInternalProfileNames().stream() + .map(name -> name + "_external") + .collect(Collectors.toSet()); + profileNames.addAll(externalProfileNames); + return profileNames.stream().map(Arguments::of); + } + + /** Test if besu will validate the combination of options within the given profile. */ + @ParameterizedTest(name = "{index} - Profile Name override: {0}") + @DisplayName("Valid Profile with overrides does not error") + @MethodSource("profileNameProvider") + public void testProfileWithNoOverrides_doesNotError(final String profileName) { + parseCommand("--profile", profileName); assertThat(commandOutput.toString(UTF_8)).isEmpty(); assertThat(commandErrorOutput.toString(UTF_8)).isEmpty(); } + + @Test + @DisplayName("Empty external profile file results in error") + public void emptyProfileFile_ShouldResultInError() { + parseCommand("--profile", "empty_external"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Unable to read from empty TOML configuration file."); + } + + @Test + @DisplayName("Non Existing profile results in error") + public void nonExistentProfileFile_ShouldResultInError() { + parseCommand("--profile", "non_existent_profile"); + + assertThat(commandOutput.toString(UTF_8)).isEmpty(); + assertThat(commandErrorOutput.toString(UTF_8)) + .contains("Unable to load external profile: non_existent_profile"); + } + + @AfterAll + public static void clearSystemProperty() { + if (originalProfilesDirProperty != null) { + System.setProperty("besu.profiles.dir", originalProfilesDirProperty); + } else { + System.clearProperty("besu.profiles.dir"); + } + } } diff --git a/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java b/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java index 0b47654f58c..983e92cec17 100644 --- a/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java +++ b/besu/src/test/java/org/hyperledger/besu/cli/TomlConfigurationDefaultProviderTest.java @@ -241,7 +241,7 @@ public void invalidConfigMustThrow(final @TempDir Path temp) throws IOException providerUnderTest.defaultValue( OptionSpec.builder("an-option").type(String.class).build())) .isInstanceOf(ParameterException.class) - .hasMessageContaining("Unable to read TOML configuration file"); + .hasMessageContaining("Unable to read from empty TOML configuration file."); } @Test diff --git a/besu/src/test/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidatesTest.java b/besu/src/test/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidatesTest.java new file mode 100644 index 00000000000..69cbf1de700 --- /dev/null +++ b/besu/src/test/java/org/hyperledger/besu/cli/config/ProfilesCompletionCandidatesTest.java @@ -0,0 +1,91 @@ +/* + * Copyright contributors to Hyperledger Besu. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + * + * SPDX-License-Identifier: Apache-2.0 + */ +package org.hyperledger.besu.cli.config; + +import org.hyperledger.besu.cli.util.ProfileFinder; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; +import java.util.Set; +import java.util.TreeSet; +import java.util.stream.Collectors; + +import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +class ProfilesCompletionCandidatesTest { + @TempDir private static Path tempProfilesDir; + private static String originalProfilesDirProperty; + + @BeforeAll + public static void copyExternalProfiles() throws IOException { + for (String internalProfileName : InternalProfileName.getInternalProfileNames()) { + final Path profilePath = tempProfilesDir.resolve(internalProfileName + "_external.toml"); + + String profileConfigFile = + InternalProfileName.valueOfIgnoreCase(internalProfileName).get().getConfigFile(); + try (InputStream resourceUrl = + ProfileFinder.class.getClassLoader().getResourceAsStream(profileConfigFile)) { + if (resourceUrl != null) { + Files.copy(resourceUrl, profilePath); + } + } + } + } + + @BeforeAll + public static void setupSystemProperty() { + originalProfilesDirProperty = System.getProperty("besu.profiles.dir"); + // sets the system property for the test + System.setProperty("besu.profiles.dir", tempProfilesDir.toString()); + } + + @Test + void profileCompletionCandidates_shouldIncludeInternalAndExternalProfiles() { + Iterator candidates = new ProfilesCompletionCandidates().iterator(); + // convert Iterator to List + List candidatesList = new ArrayList<>(); + candidates.forEachRemaining(candidatesList::add); + + Assertions.assertThat(candidatesList).containsExactlyInAnyOrderElementsOf(allProfileNames()); + } + + static Set allProfileNames() { + final Set profileNames = new TreeSet<>(InternalProfileName.getInternalProfileNames()); + final Set externalProfileNames = + InternalProfileName.getInternalProfileNames().stream() + .map(name -> name + "_external") + .collect(Collectors.toSet()); + profileNames.addAll(externalProfileNames); + return profileNames; + } + + @AfterAll + public static void clearSystemProperty() { + if (originalProfilesDirProperty != null) { + System.setProperty("besu.profiles.dir", originalProfilesDirProperty); + } else { + System.clearProperty("besu.profiles.dir"); + } + } +}