diff --git a/solr/core/src/java/org/apache/solr/cli/ApiTool.java b/solr/core/src/java/org/apache/solr/cli/ApiTool.java index 4a9c86fb8485..b04d143acd6f 100644 --- a/solr/core/src/java/org/apache/solr/cli/ApiTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ApiTool.java @@ -45,6 +45,9 @@ public class ApiTool extends ToolBase { .desc("Send a GET request to a Solr API endpoint.") .get(); + /** Parameters for the api command, independent of the command line parser. */ + record ApiParams(String getUrl, String credentials) {} + public ApiTool(ToolRuntime runtime) { super(runtime); } @@ -63,8 +66,15 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String getUrl = cli.getOptionValue(SOLR_URL_OPTION); - String response = callGet(getUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + ApiParams params = + new ApiParams( + cli.getOptionValue(SOLR_URL_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + callApi(params); + } + + void callApi(ApiParams params) throws Exception { + String response = callGet(params.getUrl(), params.credentials()); // pretty-print the response to stdout echo(response); diff --git a/solr/core/src/java/org/apache/solr/cli/AssertTool.java b/solr/core/src/java/org/apache/solr/cli/AssertTool.java index 928065f0a798..0bc56bd2209e 100644 --- a/solr/core/src/java/org/apache/solr/cli/AssertTool.java +++ b/solr/core/src/java/org/apache/solr/cli/AssertTool.java @@ -21,6 +21,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.attribute.FileOwnerAttributeView; +import java.util.ArrayList; import java.util.List; import java.util.concurrent.TimeUnit; import org.apache.commons.cli.CommandLine; @@ -40,9 +41,9 @@ */ public class AssertTool extends ToolBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); - private static String message = null; - private static boolean useExitCode = false; - private static Long timeoutMs = 1000L; + private String message = null; + private boolean useExitCode = false; + private Long timeoutMs = 1000L; private static final Option IS_NOT_ROOT_OPTION = Option.builder().desc("Asserts that we are NOT the root user.").longOpt("not-root").get(); @@ -143,6 +144,50 @@ public class AssertTool extends ToolBase { .longOpt("exitcode") .get(); + /** One requested assertion. Multiple assertions may be requested in a single invocation. */ + sealed interface Assertion { + /** Asserts that we are the root user. */ + record RootUser() implements Assertion {} + + /** Asserts that we are NOT the root user. */ + record NotRootUser() implements Assertion {} + + /** Asserts that the directory exists. */ + record DirExists(String dir) implements Assertion {} + + /** Asserts that the directory does NOT exist. */ + record DirNotExists(String dir) implements Assertion {} + + /** Asserts that we run as the same user that owns the directory. */ + record SameUser(String dir) implements Assertion {} + + /** Asserts that Solr is running on the given URL. */ + record SolrRunning(String url) implements Assertion {} + + /** Asserts that Solr is NOT running on the given URL. */ + record SolrNotRunning(String url) implements Assertion {} + + /** Asserts that Solr on the given URL is running in cloud mode. */ + record CloudMode(String url) implements Assertion {} + + /** Asserts that Solr on the given URL is NOT running in cloud mode. */ + record NotCloudMode(String url) implements Assertion {} + } + + /** + * Parameters for the assert command, independent of the command line parser. URL values are the + * raw user input; they are normalized when the assertion runs. + * + * @param credentials credentials used by the URL-based assertions, or null + * @param assertions assertions to run, in order + */ + record AssertParams( + String message, + Long timeoutMs, + boolean useExitCode, + String credentials, + List assertions) {} + public AssertTool(ToolRuntime runtime) { super(runtime); } @@ -210,49 +255,72 @@ public void runImpl(CommandLine cli) throws Exception { * @throws Exception if a tool failed, e.g. authentication failure */ protected int runAssert(CommandLine cli) throws Exception { - message = cli.getOptionValue(MESSAGE_OPTION); - timeoutMs = cli.getParsedOptionValue(TIMEOUT_OPTION, timeoutMs); - useExitCode = cli.hasOption(EXIT_CODE_OPTION); - - int ret = 0; + List assertions = new ArrayList<>(); if (cli.hasOption(IS_ROOT_OPTION)) { - ret += assertRootUser(); + assertions.add(new Assertion.RootUser()); } if (cli.hasOption(IS_NOT_ROOT_OPTION)) { - ret += assertNotRootUser(); + assertions.add(new Assertion.NotRootUser()); } if (cli.hasOption(DIRECTORY_EXISTS_OPTION)) { - ret += assertFileExists(cli.getOptionValue(DIRECTORY_EXISTS_OPTION)); + assertions.add(new Assertion.DirExists(cli.getOptionValue(DIRECTORY_EXISTS_OPTION))); } if (cli.hasOption(DIRECTORY_NOT_EXISTS_OPTION)) { - ret += assertFileNotExists(cli.getOptionValue(DIRECTORY_NOT_EXISTS_OPTION)); + assertions.add(new Assertion.DirNotExists(cli.getOptionValue(DIRECTORY_NOT_EXISTS_OPTION))); } if (cli.hasOption(SAME_USER_OPTION)) { - ret += sameUser(cli.getOptionValue(SAME_USER_OPTION)); + assertions.add(new Assertion.SameUser(cli.getOptionValue(SAME_USER_OPTION))); } if (cli.hasOption(IS_RUNNING_ON_OPTION)) { - ret += - assertSolrRunning( - cli.getOptionValue(IS_RUNNING_ON_OPTION), - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + assertions.add(new Assertion.SolrRunning(cli.getOptionValue(IS_RUNNING_ON_OPTION))); } if (cli.hasOption(IS_NOT_RUNNING_ON_OPTION)) { - ret += - assertSolrNotRunning( - cli.getOptionValue(IS_NOT_RUNNING_ON_OPTION), - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + assertions.add(new Assertion.SolrNotRunning(cli.getOptionValue(IS_NOT_RUNNING_ON_OPTION))); } if (cli.hasOption(IS_CLOUD_OPTION)) { - ret += - assertSolrRunningInCloudMode( - CLIUtils.normalizeSolrUrl(cli.getOptionValue(IS_CLOUD_OPTION)), - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + assertions.add(new Assertion.CloudMode(cli.getOptionValue(IS_CLOUD_OPTION))); } if (cli.hasOption(IS_NOT_CLOUD_OPTION)) { + assertions.add(new Assertion.NotCloudMode(cli.getOptionValue(IS_NOT_CLOUD_OPTION))); + } + return runAssert( + new AssertParams( + cli.getOptionValue(MESSAGE_OPTION), + cli.getParsedOptionValue(TIMEOUT_OPTION, timeoutMs), + cli.hasOption(EXIT_CODE_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + List.copyOf(assertions))); + } + + /** + * Runs the requested assertions. + * + * @return 0 on success, or the number of assertions that failed + * @throws Exception if an assertion failed and exit codes are not used, e.g. authentication + * failure + */ + int runAssert(AssertParams params) throws Exception { + message = params.message(); + timeoutMs = params.timeoutMs(); + useExitCode = params.useExitCode(); + String credentials = params.credentials(); + + int ret = 0; + for (Assertion assertion : params.assertions()) { ret += - assertSolrNotRunningInCloudMode( - CLIUtils.normalizeSolrUrl(cli.getOptionValue(IS_NOT_CLOUD_OPTION)), - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + switch (assertion) { + case Assertion.RootUser() -> assertRootUser(); + case Assertion.NotRootUser() -> assertNotRootUser(); + case Assertion.DirExists(String dir) -> assertFileExists(dir); + case Assertion.DirNotExists(String dir) -> assertFileNotExists(dir); + case Assertion.SameUser(String dir) -> sameUser(dir); + case Assertion.SolrRunning(String url) -> assertSolrRunning(url, credentials); + case Assertion.SolrNotRunning(String url) -> assertSolrNotRunning(url, credentials); + case Assertion.CloudMode(String url) -> + assertSolrRunningInCloudMode(CLIUtils.normalizeSolrUrl(url), credentials); + case Assertion.NotCloudMode(String url) -> + assertSolrNotRunningInCloudMode(CLIUtils.normalizeSolrUrl(url), credentials); + }; } return ret; } @@ -344,7 +412,7 @@ public int assertSolrNotRunningInCloudMode(String url, String credentials) throw return 0; } - public static int sameUser(String directory) throws Exception { + public int sameUser(String directory) throws Exception { Path path = Path.of(directory); if (Files.exists(path)) { String userForDir = userForDir(path); @@ -357,28 +425,28 @@ public static int sameUser(String directory) throws Exception { return 0; } - public static int assertFileExists(String directory) throws Exception { + public int assertFileExists(String directory) throws Exception { if (!Files.exists(Path.of(directory))) { return exitOrException("Directory " + directory + " does not exist."); } return 0; } - public static int assertFileNotExists(String directory) throws Exception { + public int assertFileNotExists(String directory) throws Exception { if (Files.exists(Path.of(directory))) { return exitOrException("Directory " + directory + " should not exist."); } return 0; } - public static int assertRootUser() throws Exception { + public int assertRootUser() throws Exception { if (!currentUser().equals("root")) { return exitOrException("Must run as root user"); } return 0; } - public static int assertNotRootUser() throws Exception { + public int assertNotRootUser() throws Exception { if (currentUser().equals("root")) { return exitOrException("Not allowed to run as root user"); } @@ -399,7 +467,7 @@ public static String userForDir(Path pathToDir) { } } - private static int exitOrException(String msg) throws AssertionFailureException { + private int exitOrException(String msg) throws AssertionFailureException { if (useExitCode) { return 1; } else { diff --git a/solr/core/src/java/org/apache/solr/cli/ClusterTool.java b/solr/core/src/java/org/apache/solr/cli/ClusterTool.java index 54626e2b7db7..f331725b1e34 100644 --- a/solr/core/src/java/org/apache/solr/cli/ClusterTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ClusterTool.java @@ -52,6 +52,9 @@ public class ClusterTool extends ToolBase { .desc("Set the property to this value.") .get(); + /** Parameters for the cluster command, independent of the command line parser. */ + record ClusterParams(String propertyName, String propertyValue, String zkHost) {} + public ClusterTool(ToolRuntime runtime) { super(runtime); } @@ -71,10 +74,18 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { + ClusterParams params = + new ClusterParams( + cli.getOptionValue(PROPERTY_OPTION), + cli.getOptionValue(VALUE_OPTION), + CLIUtils.getZkHost(cli)); + setClusterProperty(params); + } - String propertyName = cli.getOptionValue(PROPERTY_OPTION); - String propertyValue = cli.getOptionValue(VALUE_OPTION); - String zkHost = CLIUtils.getZkHost(cli); + void setClusterProperty(ClusterParams params) throws Exception { + String propertyName = params.propertyName(); + String propertyValue = params.propertyValue(); + String zkHost = params.zkHost(); if (!ZkController.checkChrootPath(zkHost, true)) { throw new IllegalStateException( diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigTool.java index a4145168165b..377cd6a44815 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigTool.java @@ -72,6 +72,15 @@ public class ConfigTool extends ToolBase { .desc("Set the property to this value; accepts JSON objects and strings.") .get(); + /** Parameters for the config command, independent of the command line parser. */ + record ConfigParams( + String solrUrl, + String action, + String collection, + String property, + String value, + String credentials) {} + public ConfigTool(ToolRuntime runtime) { super(runtime); } @@ -96,14 +105,31 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { String solrUrl = CLIUtils.normalizeSolrUrl(cli); String action = cli.getOptionValue(ACTION_OPTION, "set-property"); - String collection = cli.getOptionValue(COLLECTION_NAME_OPTION); - String property = cli.getOptionValue(PROPERTY_OPTION); String value = cli.getOptionValue(VALUE_OPTION); // value is required unless the property is one of the "unset-" type. if (!action.contains("unset-") && value == null) { throw new MissingArgumentException("'value' is a required option."); } + + ConfigParams params = + new ConfigParams( + solrUrl, + action, + cli.getOptionValue(COLLECTION_NAME_OPTION), + cli.getOptionValue(PROPERTY_OPTION), + value, + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + updateConfig(params); + } + + void updateConfig(ConfigParams params) throws Exception { + String solrUrl = params.solrUrl(); + String action = params.action(); + String collection = params.collection(); + String property = params.property(); + String value = params.value(); + Map jsonObj = new HashMap<>(); if (value != null) { Map setMap = new HashMap<>(); @@ -122,8 +148,7 @@ public void runImpl(CommandLine cli) throws Exception { echo("\nPOSTing request to Config API: " + solrUrl + updatePath); echoIfVerbose(jsonBody); - try (SolrClient solrClient = - CLIUtils.getSolrClient(solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + try (SolrClient solrClient = CLIUtils.getSolrClient(solrUrl, params.credentials())) { NamedList result = SolrCLI.postJsonToSolr(solrClient, updatePath, jsonBody); Integer statusCode = (Integer) result._get(List.of("responseHeader", "status"), null); if (statusCode == 0) { diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.java index 9fb9a23e3d2c..5af40c3dbf83 100644 --- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java @@ -138,6 +138,17 @@ public class ExportTool extends ToolBase { .desc("Comma separated list of fields to export. By default all fields are fetched.") .get(); + /** Parameters for the export command, independent of the command line parser. */ + record ExportParams( + String url, + String credentials, + String query, + String output, + String format, + boolean compress, + String fields, + String limit) {} + public ExportTool(ToolRuntime runtime) { super(runtime); } @@ -292,16 +303,25 @@ public void runImpl(CommandLine cli) throws Exception { throw new IllegalArgumentException( "Must specify a connection target via -s/--solr-connection, --solr-url, or --zk-host."); } - String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); - Info info = new MultiThreadedRunner(runtime, url, credentials); - info.query = cli.getOptionValue(QUERY_OPTION, "*:*"); - - info.setOutFormat( - cli.getOptionValue(OUTPUT_OPTION), - cli.getOptionValue(FORMAT_OPTION), - cli.hasOption(COMPRESS_OPTION)); - info.fields = cli.getOptionValue(FIELDS_OPTION); - info.setLimit(cli.getOptionValue(LIMIT_OPTION, "100")); + ExportParams params = + new ExportParams( + url, + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getOptionValue(QUERY_OPTION, "*:*"), + cli.getOptionValue(OUTPUT_OPTION), + cli.getOptionValue(FORMAT_OPTION), + cli.hasOption(COMPRESS_OPTION), + cli.getOptionValue(FIELDS_OPTION), + cli.getOptionValue(LIMIT_OPTION, "100")); + export(params); + } + + void export(ExportParams params) throws Exception { + Info info = new MultiThreadedRunner(runtime, params.url(), params.credentials()); + info.query = params.query(); + info.setOutFormat(params.output(), params.format(), params.compress()); + info.fields = params.fields(); + info.setLimit(params.limit()); info.exportDocs(); } diff --git a/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java b/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java index 176debbb0b8b..11429805028d 100644 --- a/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java +++ b/solr/core/src/java/org/apache/solr/cli/HealthcheckTool.java @@ -73,6 +73,9 @@ enum ShardState { no_leader } + /** Parameters for the healthcheck command, independent of the command line parser. */ + record HealthcheckParams(String collection, String credentials) {} + /** Requests health information about a specific collection in SolrCloud. */ public HealthcheckTool(ToolRuntime runtime) { super(runtime); @@ -85,13 +88,15 @@ public void runImpl(CommandLine cli) throws Exception { CLIO.err("Healthcheck tool only works in Solr Cloud mode."); runtime.exit(1); } + HealthcheckParams params = + new HealthcheckParams( + cli.getOptionValue(COLLECTION_NAME_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); var builder = - new HttpJettySolrClient.Builder() - .withOptionalBasicAuthCredentials( - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + new HttpJettySolrClient.Builder().withOptionalBasicAuthCredentials(params.credentials()); try (var cloudSolrClient = CLIUtils.getCloudSolrClient(solrConnection, builder)) { echoIfVerbose("Connecting to Solr at " + solrConnection.toString()); - runCloudTool(cloudSolrClient, cli); + runCloudTool(cloudSolrClient, params); } } @@ -100,8 +105,9 @@ public String getName() { return "healthcheck"; } - protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) throws Exception { - String collection = cli.getOptionValue(COLLECTION_NAME_OPTION); + protected void runCloudTool(CloudSolrClient cloudSolrClient, HealthcheckParams params) + throws Exception { + String collection = params.collection(); log.debug("Running healthcheck for {}", collection); @@ -152,13 +158,10 @@ protected void runCloudTool(CloudSolrClient cloudSolrClient, CommandLine cli) th q.setRows(0); q.set(DISTRIB, "false"); try (var solrClientForCollection = - CLIUtils.getSolrClient( - coreUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + CLIUtils.getSolrClient(coreUrl, params.credentials())) { qr = solrClientForCollection.query(q); numDocs = qr.getResults().getNumFound(); - try (var solrClient = - CLIUtils.getSolrClient( - r.getBaseUrl(), cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + try (var solrClient = CLIUtils.getSolrClient(r.getBaseUrl(), params.credentials())) { SystemInfoResponse sysResponse = (new SystemInfoRequest()).process(solrClient); uptime = SolrCLI.uptime(sysResponse.getJVMUpTimeMillis()); memory = diff --git a/solr/core/src/java/org/apache/solr/cli/PackageTool.java b/solr/core/src/java/org/apache/solr/cli/PackageTool.java index aaa3649d1be8..72b1c1ed8c4e 100644 --- a/solr/core/src/java/org/apache/solr/cli/PackageTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PackageTool.java @@ -129,149 +129,67 @@ public void runImpl(CommandLine cli) throws Exception { String cmd = cli.getArgs()[0]; - try (SolrClient solrClient = CLIUtils.getSolrClient(cli, true)) { + try (SolrClient solrClient = + CLIUtils.getSolrClient( + solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), true)) { packageManager = new PackageManager(runtime, solrClient, solrUrl, zkHost); try { repositoryManager = new RepositoryManager(solrClient, packageManager); + // Dispatches to a parser-independent method per sub-command switch (cmd) { case "add-repo": - String repoName = cli.getArgs()[1]; - String repoUrl = cli.getArgs()[2]; - repositoryManager.addRepository(repoName, repoUrl); - printGreen("Added repository: " + repoName); + addRepo(cli.getArgs()[1], cli.getArgs()[2]); break; case "add-key": - String keyFilename = cli.getArgs()[1]; - Path path = Path.of(keyFilename); - repositoryManager.addKey(Files.readAllBytes(path), path.getFileName().toString()); + addKey(Path.of(cli.getArgs()[1])); break; case "list-installed": - printGreen("Installed packages:\n-----"); - for (SolrPackageInstance pkg : packageManager.fetchInstalledPackageInstances()) { - printGreen(pkg); - } + listInstalled(); break; case "list-available": - printGreen("Available packages:\n-----"); - for (SolrPackage pkg : repositoryManager.getPackages()) { - printGreen(pkg.name + " \t\t" + pkg.description); - for (SolrPackageRelease version : pkg.versions) { - printGreen("\tVersion: " + version.version); - } - } + listAvailable(); break; case "list-deployed": if (cli.hasOption(COLLECTION_OPTION)) { - String collection = cli.getOptionValue(COLLECTION_OPTION); - Map packages = - packageManager.getPackagesDeployed(collection); - printGreen("Packages deployed on " + collection + ":"); - for (String packageName : packages.keySet()) { - printGreen("\t" + packages.get(packageName)); - } + listPackagesDeployedOnCollection(cli.getOptionValue(COLLECTION_OPTION)); } else { // nuance that we use an arg here instead of requiring a --package parameter with a - // value - // in this code path - String packageName = cli.getArgs()[1]; - Map deployedCollections = - packageManager.getDeployedCollections(packageName); - if (!deployedCollections.isEmpty()) { - printGreen("Collections on which package " + packageName + " was deployed:"); - for (String collection : deployedCollections.keySet()) { - printGreen( - "\t" - + collection - + "(" - + packageName - + ":" - + deployedCollections.get(collection) - + ")"); - } - } else { - printGreen("Package " + packageName + " not deployed on any collection."); - } + // value in this code path + listCollectionsWithPackageDeployed(cli.getArgs()[1]); } break; case "install": - { - Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1)); - String packageName = parsedVersion.first(); - String version = parsedVersion.second(); - boolean success = repositoryManager.install(packageName, version); - if (success) { - printGreen(packageName + " installed."); - } else { - printRed(packageName + " installation failed."); - } - break; - } + install(cli.getArgList().get(1)); + break; case "deploy": - { - if (cli.hasOption(CLUSTER_OPTION) || cli.hasOption(COLLECTIONS_OPTION)) { - Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1)); - String packageName = parsedVersion.first(); - String version = parsedVersion.second(); - boolean noPrompt = cli.hasOption(NO_PROMPT_OPTION); - boolean isUpdate = cli.hasOption(UPDATE_OPTION); - String[] collections = - cli.hasOption(COLLECTIONS_OPTION) - ? PackageUtils.validateCollections( - cli.getOptionValue(COLLECTIONS_OPTION).split(",")) - : new String[] {}; - String[] parameters = cli.getOptionValues(PARAM_OPTION); - packageManager.deploy( - packageName, - version, - collections, - cli.hasOption(CLUSTER_OPTION), - parameters, - isUpdate, - noPrompt); - } else { - printRed( - "Either specify --cluster to deploy cluster level plugins or --collections to deploy collection level plugins"); - } - break; + if (cli.hasOption(CLUSTER_OPTION) || cli.hasOption(COLLECTIONS_OPTION)) { + deploy( + cli.getArgList().get(1), + cli.hasOption(CLUSTER_OPTION), + cli.getOptionValue(COLLECTIONS_OPTION), + cli.getOptionValues(PARAM_OPTION), + cli.hasOption(UPDATE_OPTION), + cli.hasOption(NO_PROMPT_OPTION)); + } else { + printRed( + "Either specify --cluster to deploy cluster level plugins or --collections to deploy collection level plugins"); } + break; case "undeploy": - { - if (cli.hasOption(CLUSTER_OPTION) || cli.hasOption(COLLECTIONS_OPTION)) { - Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1)); - if (parsedVersion.second() != null) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Only package name expected, without a version. Actual: " - + cli.getArgList().get(1)); - } - String packageName = parsedVersion.first(); - String[] collections = - cli.hasOption(COLLECTIONS_OPTION) - ? PackageUtils.validateCollections( - cli.getOptionValue(COLLECTIONS_OPTION).split(",")) - : new String[] {}; - packageManager.undeploy(packageName, collections, cli.hasOption(CLUSTER_OPTION)); - } else { - printRed( - "Either specify --cluster to undeploy cluster level plugins or -collections to undeploy collection level plugins"); - } - break; + if (cli.hasOption(CLUSTER_OPTION) || cli.hasOption(COLLECTIONS_OPTION)) { + undeploy( + cli.getArgList().get(1), + cli.hasOption(CLUSTER_OPTION), + cli.getOptionValue(COLLECTIONS_OPTION)); + } else { + printRed( + "Either specify --cluster to undeploy cluster level plugins or -collections to undeploy collection level plugins"); } + break; case "uninstall": - { - Pair parsedVersion = parsePackageVersion(cli.getArgList().get(1)); - if (parsedVersion.second() == null) { - throw new SolrException( - ErrorCode.BAD_REQUEST, - "Package name and version are both required. Actual: " - + cli.getArgList().get(1)); - } - String packageName = parsedVersion.first(); - String version = parsedVersion.second(); - packageManager.uninstall(packageName, version); - break; - } + uninstall(cli.getArgList().get(1)); + break; default: throw new RuntimeException("Unrecognized command: " + cmd); } @@ -292,6 +210,122 @@ public void runImpl(CommandLine cli) throws Exception { } } + private void addRepo(String repoName, String repoUrl) throws Exception { + repositoryManager.addRepository(repoName, repoUrl); + printGreen("Added repository: " + repoName); + } + + private void addKey(Path keyFile) throws Exception { + repositoryManager.addKey(Files.readAllBytes(keyFile), keyFile.getFileName().toString()); + } + + private void listInstalled() throws Exception { + printGreen("Installed packages:\n-----"); + for (SolrPackageInstance pkg : packageManager.fetchInstalledPackageInstances()) { + printGreen(pkg); + } + } + + private void listAvailable() throws Exception { + printGreen("Available packages:\n-----"); + for (SolrPackage pkg : repositoryManager.getPackages()) { + printGreen(pkg.name + " \t\t" + pkg.description); + for (SolrPackageRelease version : pkg.versions) { + printGreen("\tVersion: " + version.version); + } + } + } + + private void listPackagesDeployedOnCollection(String collection) { + Map packages = packageManager.getPackagesDeployed(collection); + printGreen("Packages deployed on " + collection + ":"); + for (String packageName : packages.keySet()) { + printGreen("\t" + packages.get(packageName)); + } + } + + private void listCollectionsWithPackageDeployed(String packageName) { + Map deployedCollections = packageManager.getDeployedCollections(packageName); + if (!deployedCollections.isEmpty()) { + printGreen("Collections on which package " + packageName + " was deployed:"); + for (String collection : deployedCollections.keySet()) { + printGreen( + "\t" + + collection + + "(" + + packageName + + ":" + + deployedCollections.get(collection) + + ")"); + } + } else { + printGreen("Package " + packageName + " not deployed on any collection."); + } + } + + private void install(String packageNameAndVersion) throws Exception { + Pair parsedVersion = parsePackageVersion(packageNameAndVersion); + String packageName = parsedVersion.first(); + String version = parsedVersion.second(); + boolean success = repositoryManager.install(packageName, version); + if (success) { + printGreen(packageName + " installed."); + } else { + printRed(packageName + " installation failed."); + } + } + + /** + * @param collections raw comma-separated value of the --collections option, or null + */ + private void deploy( + String packageNameAndVersion, + boolean cluster, + String collections, + String[] parameters, + boolean isUpdate, + boolean noPrompt) + throws Exception { + Pair parsedVersion = parsePackageVersion(packageNameAndVersion); + String packageName = parsedVersion.first(); + String version = parsedVersion.second(); + String[] collectionArray = + collections != null + ? PackageUtils.validateCollections(collections.split(",")) + : new String[] {}; + packageManager.deploy( + packageName, version, collectionArray, cluster, parameters, isUpdate, noPrompt); + } + + /** + * @param collections raw comma-separated value of the --collections option, or null + */ + private void undeploy(String packageNameAndVersion, boolean cluster, String collections) + throws Exception { + Pair parsedVersion = parsePackageVersion(packageNameAndVersion); + if (parsedVersion.second() != null) { + throw new SolrException( + ErrorCode.BAD_REQUEST, + "Only package name expected, without a version. Actual: " + packageNameAndVersion); + } + String packageName = parsedVersion.first(); + String[] collectionArray = + collections != null + ? PackageUtils.validateCollections(collections.split(",")) + : new String[] {}; + packageManager.undeploy(packageName, collectionArray, cluster); + } + + private void uninstall(String packageNameAndVersion) throws Exception { + Pair parsedVersion = parsePackageVersion(packageNameAndVersion); + if (parsedVersion.second() == null) { + throw new SolrException( + ErrorCode.BAD_REQUEST, + "Package name and version are both required. Actual: " + packageNameAndVersion); + } + packageManager.uninstall(parsedVersion.first(), parsedVersion.second()); + } + @Override public String getHeader() { StringBuilder sb = new StringBuilder(); diff --git a/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java index dd83821ea291..04c0ca745d45 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostLogsTool.java @@ -65,6 +65,9 @@ public class PostLogsTool extends ToolBase { .desc("All files found at or below the root directory will be indexed.") .get(); + /** Parameters for the postlogs command, independent of the command line parser. */ + record PostLogsParams(String url, String rootDir, String credentials) {} + public PostLogsTool(ToolRuntime runtime) { super(runtime); } @@ -93,9 +96,16 @@ public void runImpl(CommandLine cli) throws Exception { throw new IllegalArgumentException( "Must specify a connection target via -s/--solr-connection, --solr-url, or --zk-host."); } - String rootDir = cli.getOptionValue(ROOT_DIR_OPTION); - String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); - runCommand(url, rootDir, credentials); + PostLogsParams params = + new PostLogsParams( + url, + cli.getOptionValue(ROOT_DIR_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + runCommand(params); + } + + void runCommand(PostLogsParams params) throws IOException { + runCommand(params.url(), params.rootDir(), params.credentials()); } public void runCommand(String baseUrl, String root, String credentials) throws IOException { diff --git a/solr/core/src/java/org/apache/solr/cli/PostTool.java b/solr/core/src/java/org/apache/solr/cli/PostTool.java index 4975033b8d6e..b1ed2d7dbf8f 100644 --- a/solr/core/src/java/org/apache/solr/cli/PostTool.java +++ b/solr/core/src/java/org/apache/solr/cli/PostTool.java @@ -244,6 +244,42 @@ public class PostTool extends ToolBase { mimeMap.put("log", "text/plain"); } + /** + * Options controlling how posted content and the update request are shaped. + * + * @param type content type given by the user, or null to auto-detect from file endings + * @param format {@link #FORMAT_SOLR} when the input is Solr-formatted JSON commands, else "" + * @param params raw URL-encoded {@code key=value} pairs to pass through to the update request + */ + record ContentOptions(String type, String format, String params) {} + + /** + * Options controlling traversal of directories (files mode) and links (web mode). + * + * @param fileTypes comma-separated file endings to consider + * @param delay seconds to pause between posts + * @param recursive max recursion depth, 0 to disable + */ + record CrawlOptions(String fileTypes, int delay, int recursive) {} + + /** Index maintenance actions to run after posting completes. */ + record UpdateOptions(boolean commit, boolean optimize) {} + + /** + * Parameters for the post command, independent of the command line parser. + * + * @param args positional arguments; files, directories, urls or literal data depending on mode + */ + record PostToolParams( + URI solrUpdateUrl, + String mode, + boolean dryRun, + String credentials, + String[] args, + ContentOptions content, + CrawlOptions crawl, + UpdateOptions update) {} + public PostTool(ToolRuntime runtime) { super(runtime); } @@ -273,51 +309,61 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - solrUpdateUrl = null; - if (CLIUtils.hasConnectionOption(cli)) { - String url = - CLIUtils.normalizeSolrUrl(cli) - + "/solr/" - + cli.getOptionValue(COLLECTION_NAME_OPTION) - + "/update"; - solrUpdateUrl = new URI(url); - - } else { - String url = - CLIUtils.getDefaultSolrUrl() - + "/solr/" - + cli.getOptionValue(COLLECTION_NAME_OPTION) - + "/update"; - solrUpdateUrl = new URI(url); - } + String baseUrl = + CLIUtils.hasConnectionOption(cli) + ? CLIUtils.normalizeSolrUrl(cli) + : CLIUtils.getDefaultSolrUrl(); + URI updateUrl = + new URI(baseUrl + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION) + "/update"); String mode = cli.getOptionValue(MODE_OPTION, DATA_MODE_FILES); + int defaultDelay = (mode.equals((DATA_MODE_WEB)) ? 10 : 0); - dryRun = cli.hasOption(DRY_RUN_OPTION); + PostToolParams postParams = + new PostToolParams( + updateUrl, + mode, + cli.hasOption(DRY_RUN_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getArgs(), + new ContentOptions( + cli.getOptionValue(TYPE_OPTION), + cli.hasOption(FORMAT_OPTION) + ? FORMAT_SOLR + : "", // i.e not solr formatted json commands + cli.getOptionValue(PARAMS_OPTION, "")), + new CrawlOptions( + cli.getOptionValue(FILE_TYPES_OPTION, PostTool.DEFAULT_FILE_TYPES), + cli.getParsedOptionValue(DELAY_OPTION, defaultDelay), + cli.getParsedOptionValue(RECURSIVE_OPTION, 1)), + new UpdateOptions(!cli.hasOption(SKIP_COMMIT_OPTION), cli.hasOption(OPTIMIZE_OPTION))); + postDocuments(postParams); + } + + /** Seeds the tool state from the given parameters and runs the post job. */ + void postDocuments(PostToolParams postParams) throws Exception { + solrUpdateUrl = postParams.solrUpdateUrl(); + dryRun = postParams.dryRun(); - if (cli.hasOption(TYPE_OPTION)) { - type = cli.getOptionValue(TYPE_OPTION); + if (postParams.content().type() != null) { + type = postParams.content().type(); // Turn off automatically looking up the mimetype in favour of what is passed in. auto = false; } - format = - cli.hasOption(FORMAT_OPTION) ? FORMAT_SOLR : ""; // i.e not solr formatted json commands - fileTypes = cli.getOptionValue(FILE_TYPES_OPTION, PostTool.DEFAULT_FILE_TYPES); - - int defaultDelay = (mode.equals((DATA_MODE_WEB)) ? 10 : 0); - delay = cli.getParsedOptionValue(DELAY_OPTION, defaultDelay); - recursive = cli.getParsedOptionValue(RECURSIVE_OPTION, 1); + format = postParams.content().format(); + params = postParams.content().params(); + fileTypes = postParams.crawl().fileTypes(); + delay = postParams.crawl().delay(); + recursive = postParams.crawl().recursive(); out = isVerbose() ? CLIO.getOutStream() : null; - commit = !cli.hasOption(SKIP_COMMIT_OPTION); - optimize = cli.hasOption(OPTIMIZE_OPTION); - - credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); - args = cli.getArgs(); + commit = postParams.update().commit(); + optimize = postParams.update().optimize(); - params = cli.getOptionValue(PARAMS_OPTION, ""); + credentials = postParams.credentials(); + args = postParams.args(); - execute(mode); + execute(postParams.mode()); } /** diff --git a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java index 8db32fc0b66b..05334762cb34 100644 --- a/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java +++ b/solr/core/src/java/org/apache/solr/cli/RunExampleTool.java @@ -190,6 +190,39 @@ public class RunExampleTool extends ToolBase { protected String urlScheme; private boolean usingPromptInputs = false; + /** + * Parameters consumed when starting a single Solr node via the bin/solr script, common to all + * example modes. + * + * @param extraArgs extra arguments to pass on to the {@code bin/solr start} command + */ + record StartSolrParams( + String example, + String host, + String memory, + String jvmOpts, + boolean force, + String credentials, + String extraArgs) {} + + /** + * Parameters for running a single-node example (techproducts, schemaless or films), independent + * of the command line parser. + * + * @param zkHost ZooKeeper connection string resolved from option or sysprop, or null + */ + record RunExampleParams(boolean userManaged, String zkHost, int port, StartSolrParams start) {} + + /** + * Parameters for running the multi-node cloud example, independent of the command line parser. + * + * @param promptInputs comma-separated prompt answers, or null when prompting interactively + * @param zkHost ZooKeeper connection string resolved from option or sysprop, or null + * @param basePort first node port; remaining nodes use basePort+1..+3 unless prompted otherwise + */ + record CloudExampleParams( + boolean noPrompt, String promptInputs, String zkHost, int basePort, StartSolrParams start) {} + /** Default constructor used by the framework when running as a command-line application. */ public RunExampleTool(ToolRuntime runtime) { this(null, System.in, runtime); @@ -237,14 +270,82 @@ public void runImpl(CommandLine cli) throws Exception { this.urlScheme = cli.getOptionValue(URL_SCHEME_OPTION, "http"); String exampleType = cli.getOptionValue(EXAMPLE_OPTION); - serverDir = Path.of(cli.getOptionValue(SERVER_DIR_OPTION)); + initDirs( + cli.getOptionValue(SERVER_DIR_OPTION), + cli.getOptionValue(SCRIPT_OPTION), + cli.getOptionValue(EXAMPLE_DIR_OPTION), + cli.getOptionValue(SOLR_HOME_OPTION), + exampleType); + + echoIfVerbose( + "Running with\nserverDir=" + + serverDir.toAbsolutePath() + + ",\nexampleDir=" + + exampleDir.toAbsolutePath() + + ",\nsolrHomeDir=" + + solrHomeDir.toAbsolutePath() + + "\nscript=" + + script); + + if (!"cloud".equals(exampleType) + && !"techproducts".equals(exampleType) + && !"schemaless".equals(exampleType) + && !"films".equals(exampleType)) { + throw new IllegalArgumentException( + "Unsupported example " + + exampleType + + "! Please choose one of: cloud, schemaless, techproducts, or films"); + } + + StartSolrParams startParams = + new StartSolrParams( + exampleType, + cli.getOptionValue(HOST_OPTION), + cli.getOptionValue(MEMORY_OPTION), + cli.getOptionValue(JVM_OPTS_OPTION), + cli.hasOption(FORCE_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + readExtraArgs(cli.getArgs())); + String zkHost = + CLIUtils.getCliOptionOrPropValue(cli, CommonCLIOptions.ZK_HOST_OPTION, "zkHost", null); + int port = + Integer.parseInt( + cli.getOptionValue( + PORT_OPTION, System.getenv().getOrDefault("SOLR_PORT_LISTEN", "8983"))); + + if ("cloud".equals(exampleType)) { + runCloudExample( + new CloudExampleParams( + cli.hasOption(NO_PROMPT_OPTION), + cli.getOptionValue(PROMPT_INPUTS_OPTION), + zkHost, + port, + startParams)); + } else { + runExample( + new RunExampleParams(cli.hasOption(USER_MANAGED_OPTION), zkHost, port, startParams)); + } + } + + /** + * Resolves and validates the server, example and Solr home directories plus the bin/solr script + * from the given raw values, seeding the corresponding tool state. + */ + void initDirs( + String serverDirArg, + String scriptArg, + String exampleDirArg, + String solrHomeArg, + String exampleType) + throws Exception { + serverDir = Path.of(serverDirArg); if (!Files.isDirectory(serverDir)) throw new IllegalArgumentException( "Value of --server-dir option is invalid! " + serverDir.toAbsolutePath() + " is not a directory!"); - script = cli.getOptionValue(SCRIPT_OPTION); + script = scriptArg; if (script != null) { if (!Files.isRegularFile(Path.of(script))) throw new IllegalArgumentException( @@ -263,17 +364,15 @@ public void runImpl(CommandLine cli) throws Exception { } exampleDir = - (cli.hasOption(EXAMPLE_DIR_OPTION)) - ? Path.of(cli.getOptionValue(EXAMPLE_DIR_OPTION)) - : serverDir.getParent().resolve("example"); + (exampleDirArg != null) ? Path.of(exampleDirArg) : serverDir.getParent().resolve("example"); if (!Files.isDirectory(exampleDir)) throw new IllegalArgumentException( "Value of --example-dir option is invalid! " + exampleDir.toAbsolutePath() + " is not a directory!"); - if (cli.hasOption(SOLR_HOME_OPTION)) { - solrHomeDir = Path.of(cli.getOptionValue(SOLR_HOME_OPTION)); + if (solrHomeArg != null) { + solrHomeDir = Path.of(solrHomeArg); } else { String solrHomeProp = EnvUtils.getProperty("solr.home"); if (solrHomeProp != null && !solrHomeProp.isEmpty()) { @@ -290,44 +389,19 @@ public void runImpl(CommandLine cli) throws Exception { "Value of --solr-home option is invalid! " + solrHomeDir.toAbsolutePath() + " is not a directory!"); - - echoIfVerbose( - "Running with\nserverDir=" - + serverDir.toAbsolutePath() - + ",\nexampleDir=" - + exampleDir.toAbsolutePath() - + ",\nsolrHomeDir=" - + solrHomeDir.toAbsolutePath() - + "\nscript=" - + script); - - if ("cloud".equals(exampleType)) { - runCloudExample(cli); - } else if ("techproducts".equals(exampleType) - || "schemaless".equals(exampleType) - || "films".equals(exampleType)) { - runExample(cli, exampleType); - } else { - throw new IllegalArgumentException( - "Unsupported example " - + exampleType - + "! Please choose one of: cloud, schemaless, techproducts, or films"); - } } - protected void runExample(CommandLine cli, String exampleName) throws Exception { + protected void runExample(RunExampleParams params) throws Exception { + String exampleName = params.start().example(); String collectionName = "schemaless".equals(exampleName) ? "gettingstarted" : exampleName; String configSet = "techproducts".equals(exampleName) ? "sample_techproducts_configs" : "_default"; - boolean isCloudMode = !cli.hasOption(USER_MANAGED_OPTION); - String zkHost = - CLIUtils.getCliOptionOrPropValue(cli, CommonCLIOptions.ZK_HOST_OPTION, "zkHost", null); - int port = - Integer.parseInt( - cli.getOptionValue( - PORT_OPTION, System.getenv().getOrDefault("SOLR_PORT_LISTEN", "8983"))); - Map nodeStatus = startSolr(solrHomeDir, isCloudMode, cli, port, zkHost, 30); + boolean isCloudMode = !params.userManaged(); + String zkHost = params.zkHost(); + int port = params.port(); + Map nodeStatus = + startSolr(solrHomeDir, isCloudMode, params.start(), port, zkHost, 30); String solrUrl = CLIUtils.normalizeSolrUrl((String) nodeStatus.get("baseUrl"), false); @@ -337,7 +411,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception boolean cloudMode = nodeStatus.get("cloud") != null; if (cloudMode) { if (CLIUtils.safeCheckCollectionExists( - solrUrl, collectionName, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + solrUrl, collectionName, params.start().credentials())) { alreadyExists = true; echo( "\nWARNING: Collection '" @@ -346,8 +420,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception } } else { String coreName = collectionName; - if (CLIUtils.safeCheckCoreExists( - solrUrl, coreName, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + if (CLIUtils.safeCheckCoreExists(solrUrl, coreName, params.start().credentials())) { alreadyExists = true; echo( "\nWARNING: Core '" @@ -416,9 +489,7 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception "exampledocs directory not found, skipping indexing step for the techproducts example"); } } else if ("films".equals(exampleName) && !alreadyExists) { - try (SolrClient solrClient = - CLIUtils.getSolrClient( - solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION))) { + try (SolrClient solrClient = CLIUtils.getSolrClient(solrUrl, params.start().credentials())) { echo("Adding dense vector field type to films schema"); SolrCLI.postJsonToSolr( solrClient, @@ -538,16 +609,13 @@ protected void runExample(CommandLine cli, String exampleName) throws Exception } } - protected void runCloudExample(CommandLine cli) throws Exception { + protected void runCloudExample(CloudExampleParams params) throws Exception { - usingPromptInputs = cli.hasOption(PROMPT_INPUTS_OPTION); - boolean prompt = !cli.hasOption(NO_PROMPT_OPTION); + usingPromptInputs = params.promptInputs() != null; + boolean prompt = !params.noPrompt(); int numNodes = 2; int[] cloudPorts = new int[] {8983, 7574, 8984, 7575}; - int defaultPort = - Integer.parseInt( - cli.getOptionValue( - PORT_OPTION, System.getenv().getOrDefault("SOLR_PORT_LISTEN", "8983"))); + int defaultPort = params.basePort(); if (defaultPort != 8983) { // Override the old default port numbers if user has started the example overriding // SOLR_PORT_LISTEN @@ -559,7 +627,7 @@ protected void runCloudExample(CommandLine cli) throws Exception { Scanner readInput = null; if (usingPromptInputs) { // Create a scanner from the provided prompts - String promptsValue = cli.getOptionValue(PROMPT_INPUTS_OPTION); + String promptsValue = params.promptInputs(); InputStream promptsStream = new ByteArrayInputStream(promptsValue.getBytes(StandardCharsets.UTF_8)); readInput = new Scanner(promptsStream, StandardCharsets.UTF_8); @@ -624,12 +692,11 @@ protected void runCloudExample(CommandLine cli) throws Exception { } // deal with extra args passed to the script to run the example - String zkHost = - CLIUtils.getCliOptionOrPropValue(cli, CommonCLIOptions.ZK_HOST_OPTION, "zkHost", null); + String zkHost = params.zkHost(); // start the first node (most likely with embedded ZK) Map nodeStatus = - startSolr(node1Dir.resolve("solr"), true, cli, cloudPorts[0], zkHost, 30); + startSolr(node1Dir.resolve("solr"), true, params.start(), cloudPorts[0], zkHost, 30); if (zkHost == null) { @SuppressWarnings("unchecked") @@ -648,7 +715,7 @@ protected void runCloudExample(CommandLine cli) throws Exception { startSolr( solrHomeDir.resolve("node" + (n + 1)).resolve("solr"), true, - cli, + params.start(), cloudPorts[n], zkHost, 30); @@ -662,11 +729,7 @@ protected void runCloudExample(CommandLine cli) throws Exception { // create the collection String collectionName = createCloudExampleCollection( - numNodes, - readInput, - prompt, - solrUrl, - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + numNodes, readInput, prompt, solrUrl, params.start().credentials()); echo("\n\nSolrCloud example running, please visit: " + solrUrl + " \n"); } @@ -716,25 +779,25 @@ protected void waitToSeeLiveNodes(String zkHost, int numNodes) { protected Map startSolr( Path solrHomeDir, boolean cloudMode, - CommandLine cli, + StartSolrParams params, int port, String zkHost, int maxWaitSecs) throws Exception { - String extraArgs = readExtraArgs(cli.getArgs()); + String extraArgs = params.extraArgs(); - String host = cli.getOptionValue(HOST_OPTION); - String memory = cli.getOptionValue(MEMORY_OPTION); + String host = params.host(); + String memory = params.memory(); String hostArg = (host != null && !"localhost".equals(host)) ? " --host " + host : ""; String zkHostArg = (zkHost != null) ? " -z " + zkHost : ""; String memArg = (memory != null) ? " -m " + memory : ""; String cloudModeArg = cloudMode ? "" : "--user-managed"; - String forceArg = cli.hasOption(FORCE_OPTION) ? " --force" : ""; + String forceArg = params.force() ? " --force" : ""; String verboseArg = isVerbose() ? "--verbose" : ""; - String jvmOpts = cli.getOptionValue(JVM_OPTS_OPTION); + String jvmOpts = params.jvmOpts(); String jvmOptsArg = (jvmOpts != null && !jvmOpts.isEmpty()) ? " --jvm-opts \"" + jvmOpts + "\"" : ""; @@ -752,7 +815,7 @@ protected Map startSolr( solrHome = solrHome.substring(cwdPath.length() + 1); final var syspropArg = - ("techproducts".equals(cli.getOptionValue(EXAMPLE_OPTION))) + ("techproducts".equals(params.example())) ? "-Dsolr.modules=clustering,extraction,langid,ltr,scripting -Dsolr.ltr.enabled=true -Dsolr.clustering.enabled=true" : ""; @@ -847,8 +910,7 @@ protected Map startSolr( if (code != 0) throw new Exception("Failed to start Solr using command: " + startCmdStr); } - return getNodeStatus( - solrUrl, cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), maxWaitSecs); + return getNodeStatus(solrUrl, params.credentials(), maxWaitSecs); } protected Map checkPortConflict( diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java index dfc39bf7cb2d..e7abf3ca4a7e 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotCreateTool.java @@ -44,6 +44,10 @@ public class SnapshotCreateTool extends ToolBase { .desc("Name of the snapshot to produce") .get(); + /** Parameters for the snapshot-create command, independent of the command line parser. */ + record SnapshotCreateParams( + String solrUrl, String credentials, String collectionName, String snapshotName) {} + public SnapshotCreateTool(ToolRuntime runtime) { super(runtime); } @@ -64,10 +68,18 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); - String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = CLIUtils.getSolrClient(cli)) { - createSnapshot(solrClient, collectionName, snapshotName); + SnapshotCreateParams params = + new SnapshotCreateParams( + CLIUtils.normalizeSolrUrl(cli), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getOptionValue(COLLECTION_NAME_OPTION), + cli.getOptionValue(SNAPSHOT_NAME_OPTION)); + createSnapshot(params); + } + + void createSnapshot(SnapshotCreateParams params) throws Exception { + try (var solrClient = CLIUtils.getSolrClient(params.solrUrl(), params.credentials())) { + createSnapshot(solrClient, params.collectionName(), params.snapshotName()); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java index 00b5c3c01979..de912e8e5624 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDeleteTool.java @@ -44,6 +44,10 @@ public class SnapshotDeleteTool extends ToolBase { .desc("Name of the snapshot to delete") .get(); + /** Parameters for the snapshot-delete command, independent of the command line parser. */ + record SnapshotDeleteParams( + String solrUrl, String credentials, String collectionName, String snapshotName) {} + public SnapshotDeleteTool(ToolRuntime runtime) { super(runtime); } @@ -64,10 +68,18 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); - String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = CLIUtils.getSolrClient(cli)) { - deleteSnapshot(solrClient, collectionName, snapshotName); + SnapshotDeleteParams params = + new SnapshotDeleteParams( + CLIUtils.normalizeSolrUrl(cli), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getOptionValue(COLLECTION_NAME_OPTION), + cli.getOptionValue(SNAPSHOT_NAME_OPTION)); + deleteSnapshot(params); + } + + void deleteSnapshot(SnapshotDeleteParams params) throws Exception { + try (var solrClient = CLIUtils.getSolrClient(params.solrUrl(), params.credentials())) { + deleteSnapshot(solrClient, params.collectionName(), params.snapshotName()); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java index 477ad265e7d5..e26139f65b79 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotDescribeTool.java @@ -54,6 +54,10 @@ public class SnapshotDescribeTool extends ToolBase { .desc("Name of the snapshot to describe") .get(); + /** Parameters for the snapshot-describe command, independent of the command line parser. */ + record SnapshotDescribeParams( + String solrUrl, String credentials, String collectionName, String snapshotName) {} + public SnapshotDescribeTool(ToolRuntime runtime) { super(runtime); } @@ -77,10 +81,18 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); - String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = CLIUtils.getSolrClient(cli)) { - describeSnapshot(solrClient, collectionName, snapshotName); + SnapshotDescribeParams params = + new SnapshotDescribeParams( + CLIUtils.normalizeSolrUrl(cli), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getOptionValue(COLLECTION_NAME_OPTION), + cli.getOptionValue(SNAPSHOT_NAME_OPTION)); + describeSnapshot(params); + } + + void describeSnapshot(SnapshotDescribeParams params) throws Exception { + try (var solrClient = CLIUtils.getSolrClient(params.solrUrl(), params.credentials())) { + describeSnapshot(solrClient, params.collectionName(), params.snapshotName()); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java index d71b8df8b1a2..17a2ebf7f5f0 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotExportTool.java @@ -71,6 +71,16 @@ public class SnapshotExportTool extends ToolBase { "Specifies the async request identifier to be used during snapshot export preparation.") .get(); + /** Parameters for the snapshot-export command, independent of the command line parser. */ + record SnapshotExportParams( + String solrUrl, + String credentials, + String collectionName, + String snapshotName, + String destDir, + String backupRepo, + String asyncReqId) {} + public SnapshotExportTool(ToolRuntime runtime) { super(runtime); } @@ -94,14 +104,27 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String snapshotName = cli.getOptionValue(SNAPSHOT_NAME_OPTION); - String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - String destDir = cli.getOptionValue(DEST_DIR_OPTION); - String backupRepo = cli.getOptionValue(BACKUP_REPO_NAME_OPTION); - String asyncReqId = cli.getOptionValue(ASYNC_ID_OPTION); + SnapshotExportParams params = + new SnapshotExportParams( + CLIUtils.normalizeSolrUrl(cli), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getOptionValue(COLLECTION_NAME_OPTION), + cli.getOptionValue(SNAPSHOT_NAME_OPTION), + cli.getOptionValue(DEST_DIR_OPTION), + cli.getOptionValue(BACKUP_REPO_NAME_OPTION), + cli.getOptionValue(ASYNC_ID_OPTION)); + exportSnapshot(params); + } - try (var solrClient = CLIUtils.getSolrClient(cli)) { - exportSnapshot(solrClient, collectionName, snapshotName, destDir, backupRepo, asyncReqId); + void exportSnapshot(SnapshotExportParams params) throws Exception { + try (var solrClient = CLIUtils.getSolrClient(params.solrUrl(), params.credentials())) { + exportSnapshot( + solrClient, + params.collectionName(), + params.snapshotName(), + params.destDir(), + params.backupRepo(), + params.asyncReqId()); } } diff --git a/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java index 4501fddc8397..c1444be5d5d7 100644 --- a/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java +++ b/solr/core/src/java/org/apache/solr/cli/SnapshotListTool.java @@ -37,6 +37,9 @@ public class SnapshotListTool extends ToolBase { .desc("Name of collection to list snapshots for.") .get(); + /** Parameters for the snapshot-list command, independent of the command line parser. */ + record SnapshotListParams(String solrUrl, String credentials, String collectionName) {} + public SnapshotListTool(ToolRuntime runtime) { super(runtime); } @@ -56,9 +59,17 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); - try (var solrClient = CLIUtils.getSolrClient(cli)) { - listSnapshots(solrClient, collectionName); + SnapshotListParams params = + new SnapshotListParams( + CLIUtils.normalizeSolrUrl(cli), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION), + cli.getOptionValue(COLLECTION_NAME_OPTION)); + listSnapshots(params); + } + + void listSnapshots(SnapshotListParams params) throws Exception { + try (var solrClient = CLIUtils.getSolrClient(params.solrUrl(), params.credentials())) { + listSnapshots(solrClient, params.collectionName()); } } diff --git a/solr/core/src/java/org/apache/solr/cli/StreamTool.java b/solr/core/src/java/org/apache/solr/cli/StreamTool.java index 31070e7e27b0..2d0f0dfecccc 100644 --- a/solr/core/src/java/org/apache/solr/cli/StreamTool.java +++ b/solr/core/src/java/org/apache/solr/cli/StreamTool.java @@ -22,7 +22,6 @@ import java.io.InputStream; import java.io.InputStreamReader; import java.io.LineNumberReader; -import java.io.Reader; import java.io.StringReader; import java.nio.charset.Charset; import java.nio.charset.StandardCharsets; @@ -38,6 +37,7 @@ import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; +import org.apache.solr.client.solrj.impl.CloudSolrClient; import org.apache.solr.client.solrj.io.SolrClientCache; import org.apache.solr.client.solrj.io.Tuple; import org.apache.solr.client.solrj.io.comp.StreamComparator; @@ -59,6 +59,23 @@ /** Supports stream command in the bin/solr script. */ public class StreamTool extends ToolBase { + /** + * Parameters for the stream command, independent of the command line parser. + * + * @param args positional arguments; the first entry is the streaming expression or a {@code + * .expr} file, the remaining entries substitute {@code $1}, {@code $2}, ... parameters + * @param fields raw comma-separated value of the --fields option, or null + */ + record StreamParams( + String[] args, + String execution, + String arrayDelimiter, + String delimiter, + boolean includeHeaders, + String fields, + String collection, + String credentials) {} + public StreamTool(ToolRuntime runtime) { super(runtime); } @@ -95,7 +112,7 @@ public String getUsage() { "Name of the specific collection to execute expression on if the execution is set to 'remote'. Required for 'remote' execution environment.") .get(); - private static final Option FIELDS_OPTION = + static final Option FIELDS_OPTION = Option.builder() .longOpt("fields") .argName("FIELDS") @@ -137,38 +154,57 @@ public Options getOptions() { } @Override - @SuppressWarnings({"rawtypes"}) public void runImpl(CommandLine cli) throws Exception { + StreamParams params = + new StreamParams( + cli.getArgs(), + cli.getOptionValue(EXECUTION_OPTION, "remote"), + cli.getOptionValue(ARRAY_DELIMITER_OPTION, "|"), + cli.getOptionValue(DELIMITER_OPTION, " "), + cli.hasOption(HEADER_OPTION), + cli.getOptionValue(FIELDS_OPTION), + cli.getOptionValue(COLLECTION_OPTION), + cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); + + String expr = readExpressionFromArgs(params.args()); + echoIfVerbose("Running Expression: " + expr); - String expressionArgument = cli.getArgs()[0]; - String execution = cli.getOptionValue(EXECUTION_OPTION, "remote"); - String arrayDelimiter = cli.getOptionValue(ARRAY_DELIMITER_OPTION, "|"); - String delimiter = cli.getOptionValue(DELIMITER_OPTION, " "); - boolean includeHeaders = cli.hasOption(HEADER_OPTION); - String[] outputHeaders = getOutputFields(cli); + // Validate inputs before opening any connection to Solr. + boolean local = params.execution().equalsIgnoreCase("local"); + validateExpressionArgs(local, params.collection(), expr); + + var solrConnection = CLIUtils.getSolrConnection(cli); + String solrUrl = local ? null : CLIUtils.normalizeSolrUrl(cli); + if (solrConnection == null) { + // No connection option given and none discoverable from a running Solr; fall back to the + // resolved base URL so expressions that need a Solr connection get a usable default. + solrConnection = + CloudSolrClient.CloudSolrClientConnection.parse( + solrUrl != null ? solrUrl : CLIUtils.normalizeSolrUrl(cli)); + } - LineNumberReader bufferedReader = null; - String expr; - try { - Reader inputStream = - expressionArgument.toLowerCase(Locale.ROOT).endsWith(".expr") - ? new InputStreamReader( - new FileInputStream(expressionArgument), Charset.defaultCharset()) - : new StringReader(expressionArgument); - - bufferedReader = new LineNumberReader(inputStream); - expr = StreamTool.readExpression(bufferedReader, cli.getArgs()); - echoIfVerbose("Running Expression: " + expr); - } finally { - if (bufferedReader != null) { - bufferedReader.close(); - } + runStream(params, expr, solrConnection, solrUrl); + } + + static String readExpressionFromArgs(String[] args) throws IOException { + if (args.length == 0) { + throw new IllegalArgumentException( + "A streaming expression, or a file containing one (*.expr), must be passed after the options."); } + String expressionArgument = args[0]; + try (LineNumberReader bufferedReader = + new LineNumberReader( + expressionArgument.toLowerCase(Locale.ROOT).endsWith(".expr") + ? new InputStreamReader( + new FileInputStream(expressionArgument), Charset.defaultCharset()) + : new StringReader(expressionArgument))) { + return readExpression(bufferedReader, args); + } + } - // Validate inputs before opening any connection to Solr. - boolean local = execution.equalsIgnoreCase("local"); + private static void validateExpressionArgs(boolean local, String collection, String expr) { if (!local) { - if (!cli.hasOption(COLLECTION_OPTION)) { + if (collection == null) { throw new IllegalStateException( "You must provide --name COLLECTION with --execution remote parameter."); } @@ -177,16 +213,30 @@ public void runImpl(CommandLine cli) throws Exception { "The stdin() expression is only usable with --execution local."); } } + } + + @SuppressWarnings({"rawtypes"}) + void runStream( + StreamParams params, + String expr, + CloudSolrClient.CloudSolrClientConnection solrConnection, + String solrUrl) + throws Exception { + boolean local = params.execution().equalsIgnoreCase("local"); + String arrayDelimiter = params.arrayDelimiter(); + String delimiter = params.delimiter(); + boolean includeHeaders = params.includeHeaders(); + String[] outputHeaders = getOutputFields(params.fields()); // a stream needs a context - StreamContext streamContext = createStreamContext(cli); + StreamContext streamContext = createStreamContext(solrConnection, params.credentials()); // create the stream PushBackStream pushBackStream = null; try { if (local) { pushBackStream = doLocalMode(expr, streamContext.getStreamFactory()); } else { - pushBackStream = doRemoteMode(expr, cli); + pushBackStream = doRemoteMode(expr, solrUrl, params.collection()); } pushBackStream.setStreamContext(streamContext); pushBackStream.open(); @@ -250,9 +300,9 @@ public void runImpl(CommandLine cli) throws Exception { echoIfVerbose("StreamTool -- Done."); } - private StreamContext createStreamContext(CommandLine cli) throws Exception { + private StreamContext createStreamContext( + CloudSolrClient.CloudSolrClientConnection solrConnection, String credentials) { var jettyClientBuilder = new HttpJettySolrClient.Builder(); - String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); jettyClientBuilder.withOptionalBasicAuthCredentials(credentials); HttpJettySolrClient client = jettyClientBuilder.build(); @@ -267,7 +317,6 @@ public synchronized void close() { }; try { - var solrConnection = CLIUtils.getSolrConnection(cli); echoIfVerbose("Connecting to Solr at " + solrConnection); StreamContext streamContext = new StreamContext(); @@ -315,15 +364,13 @@ private PushBackStream doLocalMode(String expr, StreamFactory streamFactory) thr * expression is running in a Solr environment. * * @param expr The streaming expression to be parsed and run remotely - * @param cli The CLI invoking the call + * @param solrUrl The base URL of the Solr node to send the expression to + * @param collection The collection to execute the expression on * @return A connection to the streaming expression that receives Tuples as they are emitted from * Solr /stream. */ - private PushBackStream doRemoteMode(String expr, CommandLine cli) throws Exception { - - String solrUrl = CLIUtils.normalizeSolrUrl(cli); - String collection = cli.getOptionValue(COLLECTION_OPTION); - + private PushBackStream doRemoteMode(String expr, String solrUrl, String collection) + throws Exception { return new PushBackStream( new SolrStream(solrUrl + "/solr/" + collection, params("qt", "/stream", "expr", expr))); } @@ -403,22 +450,22 @@ public StreamComparator getStreamSort() { } } - static String[] getOutputFields(CommandLine cli) { - if (cli.hasOption(FIELDS_OPTION)) { - - String fl = cli.getOptionValue(FIELDS_OPTION); - String[] flArray = fl.split(","); - String[] outputHeaders = new String[flArray.length]; - - for (int i = 0; i < outputHeaders.length; i++) { - outputHeaders[i] = flArray[i].trim(); - } - - return outputHeaders; - - } else { + /** + * @param fl raw comma-separated list of fields, or null + * @return the trimmed field names, or null if no fields were given + */ + static String[] getOutputFields(String fl) { + if (fl == null) { return null; } + String[] flArray = fl.split(","); + String[] outputHeaders = new String[flArray.length]; + + for (int i = 0; i < outputHeaders.length; i++) { + outputHeaders[i] = flArray[i].trim(); + } + + return outputHeaders; } public static class LocalCatStream extends CatStream { diff --git a/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java b/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java index a50254ef67e2..27ae59cddb94 100644 --- a/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/StreamToolTest.java @@ -78,7 +78,8 @@ public void testGetOutputFields() throws IOException { ToolRuntime runtime = new CLITestHelper.TestingRuntime(false); StreamTool streamTool = new StreamTool(runtime); CommandLine cli = SolrCLI.processCommandLineArgs(streamTool, args); - String[] outputFields = StreamTool.getOutputFields(cli); + String[] outputFields = + StreamTool.getOutputFields(cli.getOptionValue(StreamTool.FIELDS_OPTION)); assert outputFields != null; assertEquals(outputFields.length, 4); assertEquals(outputFields[0], "field9");