From b1c7268b6003297e6e04ef77fd0f3a5f049b88a7 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Wed, 29 Jan 2025 17:16:53 -0600 Subject: [PATCH 1/7] add changes --- solr/CHANGES.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index a66ae9098aa..e0c0ff80612 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -34,6 +34,8 @@ Improvements * SOLR-17516: `LBHttp2SolrClient` is now generic, adding support for `HttpJdkSolrClient`. (James Dyer) +* SOLR-XXXXX: Support different filesystem implementations with EmbeddedSolrServer (Andrey Bozhko) + Optimizations --------------------- * SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one. From 8cf5018487099c75e4b9543b9b5ddf9861e707e6 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Wed, 29 Jan 2025 17:17:07 -0600 Subject: [PATCH 2/7] embedded solr server works with zip filesystem --- .../schema/ManagedIndexSchemaFactory.java | 10 ++- .../TestEmbeddedSolrServerConstructors.java | 64 +++++++++++++++++++ 2 files changed, 68 insertions(+), 6 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java index df44cda12e2..ec884a026c1 100644 --- a/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java +++ b/solr/core/src/java/org/apache/solr/schema/ManagedIndexSchemaFactory.java @@ -23,7 +23,6 @@ import java.lang.invoke.MethodHandles; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.Paths; import java.util.AbstractMap.SimpleImmutableEntry; import java.util.Map.Entry; import net.jcip.annotations.NotThreadSafe; @@ -148,12 +147,11 @@ public String lookupZKManagedSchemaPath() { */ public Path lookupLocalManagedSchemaPath() { final Path legacyManagedSchemaPath = - Paths.get( - loader.getConfigPath().toString(), - ManagedIndexSchemaFactory.LEGACY_MANAGED_SCHEMA_RESOURCE_NAME); + loader + .getConfigPath() + .resolve(ManagedIndexSchemaFactory.LEGACY_MANAGED_SCHEMA_RESOURCE_NAME); - Path managedSchemaPath = - Paths.get(loader.getConfigPath().toString(), managedSchemaResourceName); + Path managedSchemaPath = loader.getConfigPath().resolve(managedSchemaResourceName); // check if we are using the legacy managed-schema file name. if (Files.exists(legacyManagedSchemaPath)) { diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java index ec6d688fe17..978b7876869 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java @@ -17,7 +17,12 @@ package org.apache.solr.client.solrj.embedded; import java.io.IOException; +import java.nio.file.FileSystems; +import java.nio.file.Files; import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.file.PathUtils; import org.apache.solr.SolrTestCaseJ4; import org.apache.solr.client.solrj.SolrQuery; import org.apache.solr.client.solrj.request.CoreAdminRequest; @@ -58,4 +63,63 @@ public void testNodeConfigConstructor() throws Exception { assertEquals(1, server.query("newcore", new SolrQuery("*:*")).getResults().getNumFound()); } } + + @SuppressWarnings("removal") + @Test + public void testPathConstructorZipFS() throws Exception { + assumeTrue( + """ + Test only works without Security Manager due to SecurityConfHandlerLocal + missing permission to read /1/2/3/4/security.json file""", + System.getSecurityManager() == null); + + Path dataDir = createTempDir("data-dir"); + Path archive = createTempFile("configset", ".zip"); + Files.delete(archive); + + // When : + // Prepare a zip archive which contains + // the configset files as shown below: + // + // configset.zip + // └── 1 + // └── 2 + // └── 3 + // └── 4 + // ├── data + // │ └── core1 + // │ ├── conf + // │ │ ├── managed-schema.xml + // │ │ └── solrconfig.xml + // │ └── core.properties + // └── solr.xml + + var zipFs = FileSystems.newFileSystem(archive, Map.of("create", "true")); + try (zipFs) { + var destDir = zipFs.getPath("1", "2", "3", "4"); + var confDir = destDir.resolve("data/core1/conf"); + Files.createDirectories(confDir); + Files.copy(TEST_PATH().resolve("solr.xml"), destDir.resolve("solr.xml")); + PathUtils.copyDirectory(configset("zipfs"), confDir); + + // Need to make sure we circumvent any Solr attempts + // to modify the archive - so to achieve that we: + // - set a custom data dir, + // - disable the update log, + // - configure the rest manager in the solrconfig.xml with InMemoryStorageIO. + Files.writeString( + confDir.resolveSibling("core.properties"), + String.join("\n", "solr.ulog.enable=false", "solr.data.dir=" + dataDir)); + } + + // Then : + // EmbeddedSolrServer successfully loads the configset directly from the archive + var configSetFs = FileSystems.newFileSystem(archive); + try (configSetFs) { + var server = new EmbeddedSolrServer(configSetFs.getPath("/1/2/3/4"), null); + try (server) { + assertEquals(List.of("core1"), server.getCoreContainer().getAllCoreNames()); + } + } + } } From 163144365f49d4e7130aa97d16ba0debed38faac Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Wed, 29 Jan 2025 17:22:18 -0600 Subject: [PATCH 3/7] update message format --- .../solrj/embedded/TestEmbeddedSolrServerConstructors.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java index 978b7876869..bc25d896b8d 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java @@ -68,9 +68,8 @@ public void testNodeConfigConstructor() throws Exception { @Test public void testPathConstructorZipFS() throws Exception { assumeTrue( - """ - Test only works without Security Manager due to SecurityConfHandlerLocal - missing permission to read /1/2/3/4/security.json file""", + "Test only works without Security Manager due to SecurityConfHandlerLocal" + + "missing permission to read /1/2/3/4/security.json file", System.getSecurityManager() == null); Path dataDir = createTempDir("data-dir"); From 7a1b0499ece25e714c1db0396765c367272b72c1 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Wed, 29 Jan 2025 19:50:24 -0600 Subject: [PATCH 4/7] add configset files --- .../configsets/zipfs/conf/managed-schema.xml | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 solr/core/src/test-files/solr/configsets/zipfs/conf/managed-schema.xml diff --git a/solr/core/src/test-files/solr/configsets/zipfs/conf/managed-schema.xml b/solr/core/src/test-files/solr/configsets/zipfs/conf/managed-schema.xml new file mode 100644 index 00000000000..61c720ee612 --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/zipfs/conf/managed-schema.xml @@ -0,0 +1,32 @@ + + + + + id + + + + + + + + + + + + From 8fbae0c4b5c1a8484815a1d4e7f0f643e1bd9073 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Wed, 29 Jan 2025 19:52:15 -0600 Subject: [PATCH 5/7] add configset files --- .../solr/configsets/zipfs/conf/solrconfig.xml | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 solr/core/src/test-files/solr/configsets/zipfs/conf/solrconfig.xml diff --git a/solr/core/src/test-files/solr/configsets/zipfs/conf/solrconfig.xml b/solr/core/src/test-files/solr/configsets/zipfs/conf/solrconfig.xml new file mode 100644 index 00000000000..933e181e35a --- /dev/null +++ b/solr/core/src/test-files/solr/configsets/zipfs/conf/solrconfig.xml @@ -0,0 +1,52 @@ + + + + + ${tests.luceneMatchVersion:LATEST} + + ${solr.data.dir:} + + + + + + + + + ${solr.ulog.dir:} + + + + + + + explicit + 10 + + + + + + org.apache.solr.rest.ManagedResourceStorage$InMemoryStorageIO + + + From e7c330cc8698b48629cb9daa263c16e2d1e66f56 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Thu, 30 Jan 2025 09:17:34 -0600 Subject: [PATCH 6/7] update changes --- solr/CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt index e0c0ff80612..81a7e0619e7 100644 --- a/solr/CHANGES.txt +++ b/solr/CHANGES.txt @@ -34,7 +34,7 @@ Improvements * SOLR-17516: `LBHttp2SolrClient` is now generic, adding support for `HttpJdkSolrClient`. (James Dyer) -* SOLR-XXXXX: Support different filesystem implementations with EmbeddedSolrServer (Andrey Bozhko) +* SOLR-17640: Support different filesystem implementations with EmbeddedSolrServer (Andrey Bozhko) Optimizations --------------------- From 5e6f1b403169656352330228c585448c4e157fa3 Mon Sep 17 00:00:00 2001 From: Andrey Bozhko Date: Thu, 30 Jan 2025 09:25:25 -0600 Subject: [PATCH 7/7] update comment --- .../embedded/TestEmbeddedSolrServerConstructors.java | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java index bc25d896b8d..825aaaa4ae1 100644 --- a/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java +++ b/solr/core/src/test/org/apache/solr/client/solrj/embedded/TestEmbeddedSolrServerConstructors.java @@ -68,7 +68,7 @@ public void testNodeConfigConstructor() throws Exception { @Test public void testPathConstructorZipFS() throws Exception { assumeTrue( - "Test only works without Security Manager due to SecurityConfHandlerLocal" + + "Test only works without Security Manager due to SecurityConfHandlerLocal " + "missing permission to read /1/2/3/4/security.json file", System.getSecurityManager() == null); @@ -102,7 +102,8 @@ public void testPathConstructorZipFS() throws Exception { PathUtils.copyDirectory(configset("zipfs"), confDir); // Need to make sure we circumvent any Solr attempts - // to modify the archive - so to achieve that we: + // to modify the archive when we point solrHome to + // the archive content. Steps to achieve that: // - set a custom data dir, // - disable the update log, // - configure the rest manager in the solrconfig.xml with InMemoryStorageIO. @@ -112,7 +113,8 @@ public void testPathConstructorZipFS() throws Exception { } // Then : - // EmbeddedSolrServer successfully loads the configset directly from the archive + // EmbeddedSolrServer successfully loads the core + // using the configset directly from the archive var configSetFs = FileSystems.newFileSystem(archive); try (configSetFs) { var server = new EmbeddedSolrServer(configSetFs.getPath("/1/2/3/4"), null);