From de6d894210e19413a5a12172d8265e64c803ddda Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 5 Jun 2026 12:28:42 +0200 Subject: [PATCH 01/19] feat(search): consumer-field contract test + index mapping health check MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Guard the denormalized search-index fields that RBAC, Data Quality, Incident Manager, Lineage, and Data Insights query (tags.tagFQN, tier, certification, owners, domains, fqnParts, testCaseStatus, ...). Renaming, retyping, or dropping one silently breaks those consumers with no compile- or boot-time failure. - SearchConsumerFields: single source of truth for the consumer contract. - SearchConsumerFieldContractTest: fails CI on a type change or a dropped field in any mapping (across all 4 languages), naming the affected consumers. - "Index Mapping Consistency" StepValidation on /system/validate: non-blocking health check that reads the live deployed mapping and warns "reindex required" when a core data-asset index is missing these fields (e.g. stale after upgrade). Adds a read-only getIndexFieldNames to the SearchClient (Elasticsearch + OpenSearch). - Fix jp/topic mapping missing top-level `domains` (found by the new test) — JP-locale topic search had lost domain-based RBAC/DQ filtering. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/SystemRepository.java | 87 ++++++++- .../service/search/IndexManagementClient.java | 13 ++ .../service/search/SearchConsumerFields.java | 107 ++++++++++++ .../service/search/SearchRepository.java | 5 + .../elasticsearch/ElasticSearchClient.java | 5 + .../ElasticSearchIndexManager.java | 24 +++ .../search/opensearch/OpenSearchClient.java | 5 + .../opensearch/OpenSearchIndexManager.java | 24 +++ ...ystemRepositoryMappingConsistencyTest.java | 119 +++++++++++++ .../SearchConsumerFieldContractTest.java | 165 ++++++++++++++++++ .../elasticsearch/jp/topic_index_mapping.json | 47 +++++ 11 files changed, 600 insertions(+), 1 deletion(-) create mode 100644 openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/search/SearchConsumerFieldContractTest.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index 5cfc44dd2ceb..dd25c6d76bca 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -22,6 +22,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.UUID; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -66,6 +67,7 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.schema.utils.ResultList; import org.openmetadata.sdk.PipelineServiceClientInterface; +import org.openmetadata.search.IndexMapping; import org.openmetadata.service.Entity; import org.openmetadata.service.OpenMetadataApplicationConfig; import org.openmetadata.service.exception.CustomExceptionMessage; @@ -77,6 +79,7 @@ import org.openmetadata.service.logstorage.LogStorageInterface; import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.SearchConsumerFields; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.vector.client.EmbeddingClient; import org.openmetadata.service.secrets.SecretsManager; @@ -108,6 +111,7 @@ public class SystemRepository { private static final String FAILED_TO_UPDATE_SETTINGS = "Failed to Update Settings {}"; public static final String INTERNAL_SERVER_ERROR_WITH_REASON = "Internal Server Error. Reason :"; private static final String VECTOR_EMBEDDING_INDEX_KEY = "vectorEmbedding"; + private static final String MAPPING_CONSISTENCY_VALIDATION_KEY = "Index Mapping Consistency"; private final SystemDAO dao; private final MigrationValidationClient migrationValidationClient; @@ -116,7 +120,10 @@ private enum ValidationStepDescription { SEARCH("Validate that the search client is available."), PIPELINE_SERVICE_CLIENT("Validate that the pipeline service client is available."), JWT_TOKEN("Validate that the ingestion-bot JWT token can be properly decoded."), - MIGRATION("Validate that all the necessary migrations have been properly executed."); + MIGRATION("Validate that all the necessary migrations have been properly executed."), + SEARCH_MAPPING( + "Validate that deployed search indexes expose the denormalized fields used by RBAC, " + + "Data Quality, Incidents, Lineage, and Data Insights."); public final String key; @@ -555,6 +562,9 @@ public ValidationResponse validateSystem( "Semantic Search", getEmbeddingsValidation(applicationConfig)); } + validation.setAdditionalProperty( + MAPPING_CONSISTENCY_VALIDATION_KEY, getMappingConsistencyValidation()); + addExtraValidations(applicationConfig, validation); return validation; @@ -858,6 +868,81 @@ List findMissingIndexes(SearchRepository searchRepository) { return missing; } + private StepValidation getMappingConsistencyValidation() { + StepValidation step = + new StepValidation().withDescription(ValidationStepDescription.SEARCH_MAPPING.key); + SearchRepository searchRepository = Entity.getSearchRepository(); + StepValidation result; + if (searchRepository.getSearchClient().isClientAvailable()) { + List inconsistent = findIndexesWithMissingConsumerFields(searchRepository); + result = + step.withPassed(inconsistent.isEmpty()) + .withMessage(buildMappingConsistencyMessage(inconsistent)); + } else { + result = + step.withPassed(Boolean.TRUE).withMessage("Skipped: search instance is not reachable."); + } + return result; + } + + private String buildMappingConsistencyMessage(List inconsistent) { + String message; + if (inconsistent.isEmpty()) { + message = + "All core data asset indexes expose the fields used by RBAC, Data Quality, Incidents, " + + "Lineage, and Data Insights."; + } else { + message = + String.format( + "WARNING: %d index(es) are missing denormalized fields that RBAC, Data Quality, " + + "Incidents, Lineage, and Data Insights depend on — a reindex is required to " + + "restore them. Affected: %s", + inconsistent.size(), inconsistent); + } + return message; + } + + @VisibleForTesting + List findIndexesWithMissingConsumerFields(SearchRepository searchRepository) { + List inconsistent = new ArrayList<>(); + try { + Map indexMap = searchRepository.getEntityIndexMap(); + for (String entityType : SearchConsumerFields.CANARY_DATA_ASSET_ENTITIES) { + collectMissingConsumerFields( + searchRepository, entityType, indexMap.get(entityType), inconsistent); + } + } catch (Exception e) { + LOG.warn("Failed to validate index mapping consistency: {}", e.getMessage()); + } + return inconsistent; + } + + private void collectMissingConsumerFields( + SearchRepository searchRepository, + String entityType, + IndexMapping indexMapping, + List inconsistent) { + if (indexMapping != null) { + Set liveFields = searchRepository.getIndexFieldNames(indexMapping); + if (!liveFields.isEmpty()) { + List missing = missingConsumerFields(liveFields); + if (!missing.isEmpty()) { + inconsistent.add(entityType + " (missing: " + missing + ")"); + } + } + } + } + + private List missingConsumerFields(Set liveFields) { + List missing = new ArrayList<>(); + for (String required : SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS) { + if (!liveFields.contains(required)) { + missing.add(required); + } + } + return missing; + } + private boolean validateDataInsights() { boolean isValid = false; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java index 64fe48045c51..53ffa040f0c5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java @@ -207,4 +207,17 @@ default long getDocumentCount(String indexName) { } return 0; } + + /** + * Get the top-level field (property) names declared in an index's live mapping. Used by the + * "Index Mapping Consistency" health check to detect deployed indexes missing the denormalized + * fields product features rely on (e.g. left stale after an upgrade without a reindex). + * + * @param indexName the index (or alias) name to inspect + * @return the set of top-level field names, or an empty set if the index is unreachable/missing + */ + default Set getIndexFieldNames(String indexName) { + throw new UnsupportedOperationException( + "getIndexFieldNames is not implemented for this search client"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java new file mode 100644 index 000000000000..26f0cc6b0250 --- /dev/null +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java @@ -0,0 +1,107 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import java.util.List; +import java.util.Set; + +/** + * Single source of truth for the denormalized search-index fields that product features beyond + * Explore depend on. RBAC, Data Quality, Incident Manager, Lineage, and Data Insights all query + * these fields directly, so renaming, retyping, or dropping one of them in a mapping silently + * breaks those consumers (empty facets, wrong access-control results, zeroed Data Quality widgets) + * with no compile-time or boot-time failure. + * + *

Two guards reference this class so the contract lives in one place: + * + *

    + *
  • {@code SearchConsumerFieldContractTest} fails CI when a shipped mapping file violates it. + *
  • {@code SystemRepository}'s "Index Mapping Consistency" health check warns when a deployed + * index is missing these fields (e.g. left stale after an upgrade without a reindex). + *
+ */ +public final class SearchConsumerFields { + private SearchConsumerFields() {} + + /** + * A denormalized leaf field and the type it must keep. Validated "where present" — a mapping that + * does not define the path is out of scope (e.g. {@code testCaseResult.testCaseStatus} exists + * only on the test-case indexes). Catches the dangerous silent break: a retype (keyword to text) + * that kills the aggregations, sorts, and term filters every consumer builds on these fields. + */ + public record ConsumerField(String path, String expectedType, String consumers) {} + + public static final List TYPED_LEAF_FIELDS = + List.of( + new ConsumerField( + "tags.tagFQN", + "keyword", + "RBAC tag rules, Explore tag facet, Data Quality tag filter, column-lineage tags"), + new ConsumerField( + "tier.tagFQN", + "keyword", + "RBAC Tier rules, Explore tier facet, Data Quality tier widgets, Data Insights tier"), + new ConsumerField( + "certification.tagLabel.tagFQN", + "keyword", + "RBAC certification rules, Explore and Data Quality certification filters"), + new ConsumerField( + "classificationTags", "keyword", "Explore classification facet, tag aggregations"), + new ConsumerField("glossaryTags", "keyword", "Explore glossary facet, tag aggregations"), + new ConsumerField( + "domains.fullyQualifiedName", + "keyword", + "RBAC domain rules, Data Quality and Incident domain filters, domain asset counts"), + new ConsumerField( + "fqnParts", "keyword", "Search autocomplete/suggest, hierarchical search"), + new ConsumerField( + "testCaseResult.testCaseStatus", + "keyword", + "Test Suite execution summary, Data Quality status filter"), + new ConsumerField( + "testCaseResolutionStatusType", + "keyword", + "Incident Manager listing and aggregations")); + + /** + * Core data-asset entity types verified to expose the full denormalized field set at the top + * level. Used as a canary: the shared doc-builder pipeline writes these fields into every data + * asset, so if it drops one it drops from all of these at once. {@code table} is intentionally + * excluded — its static mapping does not declare {@code fqnParts} top-level (it is dynamically + * mapped), which would be a false positive for that one field. + */ + public static final Set CANARY_DATA_ASSET_ENTITIES = + Set.of( + "dashboard", + "pipeline", + "topic", + "container", + "mlmodel", + "dashboardDataModel", + "searchIndex", + "apiEndpoint", + "metric", + "chart"); + + /** Top-level denormalized fields every {@link #CANARY_DATA_ASSET_ENTITIES} index must expose. */ + public static final Set CANARY_REQUIRED_TOP_LEVEL_FIELDS = + Set.of( + "fqnParts", + "tier", + "tags", + "owners", + "domains", + "certification", + "classificationTags", + "glossaryTags"); +} diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 64cbaebd28d7..00d45368f1c4 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -734,6 +734,11 @@ public boolean indexExists(IndexMapping indexMapping) { return !searchClient.getIndicesByAlias(indexName).isEmpty(); } + public Set getIndexFieldNames(IndexMapping indexMapping) { + String indexName = indexMapping.getIndexName(clusterAlias); + return searchClient.getIndexFieldNames(indexName); + } + public void createIndex(IndexMapping indexMapping) { try { String indexName = indexMapping.getIndexName(clusterAlias); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index abb2d2d0050a..05312003b883 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -217,6 +217,11 @@ public boolean indexExists(String indexName) { return indexManager.indexExists(indexName); } + @Override + public Set getIndexFieldNames(String indexName) { + return indexManager.getIndexFieldNames(indexName); + } + @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { indexManager.createIndex(indexMapping, indexMappingContent); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java index 0f32214bf507..4f3ce58d21fb 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java @@ -11,6 +11,7 @@ import es.co.elastic.clients.elasticsearch.indices.ForcemergeResponse; import es.co.elastic.clients.elasticsearch.indices.GetAliasRequest; import es.co.elastic.clients.elasticsearch.indices.GetAliasResponse; +import es.co.elastic.clients.elasticsearch.indices.GetMappingResponse; import es.co.elastic.clients.elasticsearch.indices.PutIndicesSettingsRequest; import es.co.elastic.clients.elasticsearch.indices.PutIndicesSettingsResponse; import es.co.elastic.clients.elasticsearch.indices.PutMappingRequest; @@ -63,6 +64,29 @@ public boolean indexExists(String indexName) { } } + @Override + public Set getIndexFieldNames(String indexName) { + Set fieldNames = new HashSet<>(); + if (isClientAvailable) { + try { + GetMappingResponse response = client.indices().getMapping(g -> g.index(indexName)); + response + .mappings() + .values() + .forEach( + mappingRecord -> { + if (mappingRecord.mappings() != null + && mappingRecord.mappings().properties() != null) { + fieldNames.addAll(mappingRecord.mappings().properties().keySet()); + } + }); + } catch (Exception e) { + LOG.warn("Failed to get field names for index {}: {}", indexName, e.getMessage()); + } + } + return fieldNames; + } + @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { if (!isClientAvailable) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index 930852c59119..e251df36f48e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -204,6 +204,11 @@ public boolean indexExists(String indexName) { return indexManager.indexExists(indexName); } + @Override + public Set getIndexFieldNames(String indexName) { + return indexManager.getIndexFieldNames(indexName); + } + @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { indexManager.createIndex(indexMapping, indexMappingContent); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java index 629d0121c996..ed3e5d797484 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java @@ -25,6 +25,7 @@ import os.org.opensearch.client.opensearch.indices.ForcemergeResponse; import os.org.opensearch.client.opensearch.indices.GetAliasRequest; import os.org.opensearch.client.opensearch.indices.GetAliasResponse; +import os.org.opensearch.client.opensearch.indices.GetMappingResponse; import os.org.opensearch.client.opensearch.indices.IndexSettings; import os.org.opensearch.client.opensearch.indices.PutIndicesSettingsRequest; import os.org.opensearch.client.opensearch.indices.PutIndicesSettingsResponse; @@ -66,6 +67,29 @@ public boolean indexExists(String indexName) { } } + @Override + public Set getIndexFieldNames(String indexName) { + Set fieldNames = new HashSet<>(); + if (isClientAvailable) { + try { + GetMappingResponse response = client.indices().getMapping(g -> g.index(indexName)); + response + .result() + .values() + .forEach( + mappingRecord -> { + if (mappingRecord.mappings() != null + && mappingRecord.mappings().properties() != null) { + fieldNames.addAll(mappingRecord.mappings().properties().keySet()); + } + }); + } catch (Exception e) { + LOG.warn("Failed to get field names for index {}: {}", indexName, e.getMessage()); + } + } + return fieldNames; + } + @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { if (!isClientAvailable) { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java new file mode 100644 index 000000000000..f911befd22a4 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java @@ -0,0 +1,119 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.jdbi3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; +import org.openmetadata.search.IndexMapping; +import org.openmetadata.service.Entity; +import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; +import org.openmetadata.service.migration.MigrationValidationClient; +import org.openmetadata.service.search.SearchConsumerFields; +import org.openmetadata.service.search.SearchRepository; + +class SystemRepositoryMappingConsistencyTest { + + private MockedStatic entityMock; + private MockedStatic migrationMock; + private SearchRepository searchRepository; + private SystemRepository systemRepository; + + @BeforeEach + void setup() { + entityMock = mockStatic(Entity.class); + migrationMock = mockStatic(MigrationValidationClient.class); + + CollectionDAO collectionDAO = mock(CollectionDAO.class); + SystemDAO systemDAO = mock(SystemDAO.class); + when(collectionDAO.systemDAO()).thenReturn(systemDAO); + entityMock.when(Entity::getCollectionDAO).thenReturn(collectionDAO); + + MigrationValidationClient migrationClient = mock(MigrationValidationClient.class); + migrationMock.when(MigrationValidationClient::getInstance).thenReturn(migrationClient); + + searchRepository = mock(SearchRepository.class); + entityMock.when(Entity::getSearchRepository).thenReturn(searchRepository); + + systemRepository = new SystemRepository(); + } + + @AfterEach + void tearDown() { + entityMock.close(); + migrationMock.close(); + } + + @Test + void testAllRequiredFieldsPresentReturnsEmpty() { + IndexMapping dashboard = mock(IndexMapping.class); + when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("dashboard", dashboard)); + when(searchRepository.getIndexFieldNames(dashboard)) + .thenReturn(SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS); + + List inconsistent = + systemRepository.findIndexesWithMissingConsumerFields(searchRepository); + + assertTrue(inconsistent.isEmpty()); + } + + @Test + void testMissingFieldIsReportedWithEntityAndField() { + IndexMapping dashboard = mock(IndexMapping.class); + Set partial = new HashSet<>(SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS); + partial.remove("tier"); + when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("dashboard", dashboard)); + when(searchRepository.getIndexFieldNames(dashboard)).thenReturn(partial); + + List inconsistent = + systemRepository.findIndexesWithMissingConsumerFields(searchRepository); + + assertEquals(1, inconsistent.size()); + assertTrue(inconsistent.get(0).contains("dashboard")); + assertTrue(inconsistent.get(0).contains("tier")); + } + + @Test + void testEmptyLiveFieldsSkippedSoMissingIndexIsNotDoubleReported() { + IndexMapping dashboard = mock(IndexMapping.class); + when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("dashboard", dashboard)); + when(searchRepository.getIndexFieldNames(dashboard)).thenReturn(Set.of()); + + List inconsistent = + systemRepository.findIndexesWithMissingConsumerFields(searchRepository); + + assertTrue(inconsistent.isEmpty()); + } + + @Test + void testNonCanaryEntityIsIgnored() { + IndexMapping tag = mock(IndexMapping.class); + when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("tag", tag)); + + List inconsistent = + systemRepository.findIndexesWithMissingConsumerFields(searchRepository); + + assertTrue(inconsistent.isEmpty()); + } +} diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchConsumerFieldContractTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchConsumerFieldContractTest.java new file mode 100644 index 000000000000..03e9f8d0e973 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchConsumerFieldContractTest.java @@ -0,0 +1,165 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.search; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.openmetadata.schema.utils.JsonUtils; +import org.openmetadata.search.IndexMapping; +import org.openmetadata.search.IndexMappingLoader; +import org.openmetadata.service.search.SearchConsumerFields.ConsumerField; + +/** + * Guards the search-index "consumer contract": the denormalized fields that RBAC, Data Quality, + * Incidents, Lineage, and Data Insights query. A mapping change that renames, retypes, or drops one + * of these fields breaks those features silently at runtime — this test turns that into a CI + * failure that names the affected consumers. Source of truth: {@link SearchConsumerFields}. + */ +class SearchConsumerFieldContractTest { + + private static final List LANGUAGES = List.of("en", "jp", "ru", "zh"); + private static Map allMappings; + + @BeforeAll + static void loadAllMappings() throws IOException { + IndexMappingLoader.init(); + IndexMappingLoader loader = IndexMappingLoader.getInstance(); + allMappings = new HashMap<>(); + for (Map.Entry entry : loader.getIndexMapping().entrySet()) { + String entity = entry.getKey(); + IndexMapping indexMapping = entry.getValue(); + for (String language : LANGUAGES) { + String filePath = indexMapping.getIndexMappingFile(language); + try (InputStream in = + SearchConsumerFieldContractTest.class.getClassLoader().getResourceAsStream(filePath)) { + if (in == null) { + continue; + } + String key = entity + "[" + language + "]"; + allMappings.put( + key, JsonUtils.readTree(new String(in.readAllBytes(), StandardCharsets.UTF_8))); + } + } + } + assertTrue(allMappings.size() > 1, "Should load more than one index mapping"); + } + + @Test + void consumerLeafFieldsKeepTheirTypeWherePresent() { + List violations = new ArrayList<>(); + for (Map.Entry entry : allMappings.entrySet()) { + JsonNode properties = topLevelProperties(entry.getValue()); + if (properties == null) { + continue; + } + for (ConsumerField field : SearchConsumerFields.TYPED_LEAF_FIELDS) { + collectTypeViolation(entry.getKey(), properties, field, violations); + } + } + assertTrue( + violations.isEmpty(), + "Search consumer fields changed type. These fields are queried by RBAC, Data Quality, " + + "Incidents, Lineage, and Data Insights; retyping them silently breaks aggregations, " + + "term filters, and access control. Violations:\n" + + String.join("\n", violations)); + } + + @Test + void coreDataAssetIndexesExposeConsumerFields() { + List violations = new ArrayList<>(); + for (Map.Entry entry : allMappings.entrySet()) { + String entity = entry.getKey().substring(0, entry.getKey().indexOf('[')); + if (SearchConsumerFields.CANARY_DATA_ASSET_ENTITIES.contains(entity)) { + collectMissingFieldViolations(entry.getKey(), entry.getValue(), violations); + } + } + assertTrue( + violations.isEmpty(), + "Core data asset indexes are missing denormalized fields that the shared doc-builder " + + "pipeline is expected to write. Dropping these breaks Explore facets, RBAC, Data " + + "Quality, and Data Insights at once. Violations:\n" + + String.join("\n", violations)); + } + + private static void collectTypeViolation( + String mappingKey, JsonNode properties, ConsumerField field, List violations) { + JsonNode node = findField(properties, field.path()); + if (node != null) { + String type = node.path("type").asText(""); + if (!field.expectedType().equals(type)) { + String actual = type.isEmpty() ? "missing \"type\" (implicit object)" : "\"" + type + "\""; + violations.add( + mappingKey + + " " + + field.path() + + ": expected \"" + + field.expectedType() + + "\" but found " + + actual + + " — breaks: " + + field.consumers()); + } + } + } + + private static void collectMissingFieldViolations( + String mappingKey, JsonNode root, List violations) { + JsonNode properties = topLevelProperties(root); + assertNotNull( + properties, + "Index mapping for '" + + mappingKey + + "' has no properties — mapping file may be malformed."); + for (String required : SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS) { + if (!properties.has(required)) { + violations.add(mappingKey + " missing top-level field \"" + required + "\""); + } + } + } + + private static JsonNode findField(JsonNode rootProperties, String dottedPath) { + String[] segments = dottedPath.split("\\."); + JsonNode properties = rootProperties; + JsonNode found = null; + for (int i = 0; i < segments.length && properties != null && !properties.isMissingNode(); i++) { + JsonNode node = properties.path(segments[i]); + if (node.isMissingNode()) { + properties = null; + } else if (i == segments.length - 1) { + found = node; + } else { + properties = node.path("properties"); + } + } + return found; + } + + private static JsonNode topLevelProperties(JsonNode root) { + JsonNode properties = root.path("mappings").path("properties"); + if (properties.isMissingNode()) { + properties = root.path("properties"); + } + return properties.isMissingNode() ? null : properties; + } +} diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json index f0bb7f16a148..198bae38bcc8 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -387,6 +387,53 @@ "type": "text", "analyzer": "om_analyzer" }, + "domains": { + "properties": { + "id": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 36 + } + } + }, + "type": { + "type": "keyword" + }, + "name": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "ignore_above": 256 + } + } + }, + "displayName": { + "type": "keyword", + "fields": { + "keyword": { + "type": "keyword", + "normalizer": "lowercase_normalizer", + "ignore_above": 256 + } + } + }, + "fullyQualifiedName": { + "type": "keyword" + }, + "description": { + "type": "text" + }, + "deleted": { + "type": "boolean" + }, + "href": { + "type": "text" + } + } + }, "cleanupPolicies": { "type": "keyword" }, From 4b56031bef1f8aa98b96c48ba424e00b4a1cde2a Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 5 Jun 2026 15:13:03 +0200 Subject: [PATCH 02/19] test(search): per-language behavioral IT for the search consumer contract MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Index a document into the real en/jp/ru/zh index mapping for topic and test_case, then run the actual search/filter/aggregation each feature uses (tag/tier/certification/domain term filters, owners nested RBAC query, testCaseResult.testCaseStatus aggregation, fqnParts term) and assert it returns the document. A failure reads as a broken feature in a specific language, not "index is missing a mapping" — and would have caught the jp/topic domains drop. Runs against a real OpenSearch testcontainer. Text analyzers are stripped before index creation (the consumer fields are keyword/nested and analyzer-independent) so each language index is creatable without analysis plugins while the real field structure is still validated per language. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/SearchConsumerFieldBehaviorIT.java | 373 ++++++++++++++++++ 1 file changed, 373 insertions(+) create mode 100644 openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java new file mode 100644 index 000000000000..bcc1c116e4c2 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java @@ -0,0 +1,373 @@ +package org.openmetadata.it.tests; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ObjectNode; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import org.apache.hc.core5.http.HttpHost; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.openmetadata.service.search.opensearch.OsUtils; +import org.opensearch.testcontainers.OpensearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import org.testcontainers.utility.DockerImageName; +import os.org.opensearch.client.json.jackson.JacksonJsonpMapper; +import os.org.opensearch.client.opensearch.OpenSearchClient; +import os.org.opensearch.client.opensearch.generic.Requests; +import os.org.opensearch.client.transport.httpclient5.ApacheHttpClient5Transport; +import os.org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; + +/** + * Behavioral, per-language guard for the search "consumer contract": rather than asserting a + * mapping declares a field, this indexes a real document into the actual index mapping for + * every supported language ({@code en/jp/ru/zh}) and then runs the actual search / filter / + * aggregation each product feature relies on, asserting it returns the document. A failure is + * reported as a broken feature in a specific language (e.g. "Domain filter is broken in + * [jp]"), which is what an operator can act on — not "index is missing a mapping". + * + *

This is the test that would have caught the {@code jp/topic} mapping dropping top-level {@code + * domains}: the domain-filter assertion below returns zero hits for {@code jp} until the mapping is + * fixed. The earlier structural unit test could not, because it only knew which fields to look for + * from a hand-maintained list. + * + *

The query types mirror real consumers: a nested query for {@code owners} (RBAC isOwner), term + * filters for tags/tier/certification/domains (Explore facets, RBAC, Data Quality), a terms + * aggregation for {@code testCaseResult.testCaseStatus} (the Data Quality execution summary), and a + * keyword term on {@code fqnParts} (hierarchical search / autocomplete). + */ +@Testcontainers +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class SearchConsumerFieldBehaviorIT { + + private static final List LANGUAGES = List.of("en", "jp", "ru", "zh"); + + private static final String TAG_FQN = "PII.Sensitive"; + private static final String TIER_FQN = "Tier.Tier1"; + private static final String CERTIFICATION_FQN = "Certification.Gold"; + private static final String DOMAIN_FQN = "Finance"; + private static final String FQN_PART = "svc.ns"; + private static final String OWNER_ID = "11111111-1111-1111-1111-111111111111"; + private static final String DOMAIN_ID = "22222222-2222-2222-2222-222222222222"; + private static final String TOPIC_ID = "33333333-3333-3333-3333-333333333333"; + private static final String TOPIC_FQN = "svc.ns.orders"; + private static final String TEST_CASE_ID = "44444444-4444-4444-4444-444444444444"; + private static final String TEST_CASE_FQN = "svc.ns.orders.columns.amount.notNull"; + private static final String TEST_CASE_STATUS = "Success"; + + @Container + static OpensearchContainer opensearch = + new OpensearchContainer<>(DockerImageName.parse("opensearchproject/opensearch:3.4.0")) + .withStartupTimeout(Duration.ofMinutes(5)) + .withEnv("discovery.type", "single-node") + .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "Test@12345") + .withEnv("DISABLE_SECURITY_PLUGIN", "true") + .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") + .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m"); + + private OpenSearchClient openSearchClient; + private ObjectMapper mapper; + + @BeforeAll + void setUp() throws Exception { + HttpHost httpHost = new HttpHost("http", opensearch.getHost(), opensearch.getMappedPort(9200)); + ApacheHttpClient5Transport transport = + ApacheHttpClient5TransportBuilder.builder(httpHost) + .setMapper(new JacksonJsonpMapper()) + .build(); + openSearchClient = new OpenSearchClient(transport); + mapper = new ObjectMapper(); + + for (String language : LANGUAGES) { + createIndex(topicIndex(language), "/elasticsearch/" + language + "/topic_index_mapping.json"); + indexDocument(topicIndex(language), TOPIC_ID, topicDocument()); + createIndex( + testCaseIndex(language), "/elasticsearch/" + language + "/test_case_index_mapping.json"); + indexDocument(testCaseIndex(language), TEST_CASE_ID, testCaseDocument()); + } + } + + @AfterAll + void tearDown() throws Exception { + if (openSearchClient != null) { + openSearchClient._transport().close(); + } + } + + @Test + void tagFilterReturnsAssetInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Tag filter (Explore tag facet, Data Quality tag filter, RBAC tag rules)", + language -> hits(topicIndex(language), termQuery("tags.tagFQN", TAG_FQN)) == 1); + } + + @Test + void tierAggregationHasBucketInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Tier aggregation (Explore tier facet, Data Quality tier widget)", + language -> + bucketKeys(topicIndex(language), termsAggregation("tier", "tier.tagFQN"), "tier") + .contains(TIER_FQN)); + } + + @Test + void certificationFilterReturnsAssetInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Certification filter (Explore + Data Quality certification filter, RBAC certification)", + language -> + hits( + topicIndex(language), + termQuery("certification.tagLabel.tagFQN", CERTIFICATION_FQN)) + == 1); + } + + @Test + void ownerNestedRbacQueryReturnsAssetInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Owner RBAC nested query (isOwner access control, owners facet)", + language -> + hits(topicIndex(language), nestedTermQuery("owners", "owners.id", OWNER_ID)) == 1); + } + + @Test + void domainFilterReturnsAssetInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Domain filter (RBAC hasDomain, Data Quality/Incident domain filter, domain asset counts)", + language -> + hits(topicIndex(language), termQuery("domains.fullyQualifiedName", DOMAIN_FQN)) == 1); + } + + @Test + void fqnPartsTermReturnsAssetInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Hierarchical search / autocomplete (fqnParts)", + language -> hits(topicIndex(language), termQuery("fqnParts", FQN_PART)) == 1); + } + + @Test + void dataQualityStatusAggregationHasBucketInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Data Quality status aggregation (Test Suite execution summary, DQ status filter)", + language -> + bucketKeys( + testCaseIndex(language), + termsAggregation("status", "testCaseResult.testCaseStatus"), + "status") + .contains(TEST_CASE_STATUS)); + } + + @FunctionalInterface + private interface LanguageProbe { + boolean passes(String language) throws Exception; + } + + private void assertFeatureWorksInAllLanguages(String feature, LanguageProbe probe) + throws Exception { + List broken = new ArrayList<>(); + for (String language : LANGUAGES) { + if (!probe.passes(language)) { + broken.add(language); + } + } + assertTrue( + broken.isEmpty(), + feature + + " is broken in language(s): " + + broken + + ". The search/filter/aggregation this feature relies on did not return the indexed " + + "asset for those languages — the index mapping or analyzer for that language does not " + + "support the query (missing field, wrong type, or wrong analyzer)."); + } + + private int hits(String index, String queryBody) throws Exception { + return runSearch(index, queryBody).path("hits").path("hits").size(); + } + + private List bucketKeys(String index, String aggBody, String aggName) throws Exception { + List keys = new ArrayList<>(); + JsonNode buckets = runSearch(index, aggBody).path("aggregations").path(aggName).path("buckets"); + for (JsonNode bucket : buckets) { + keys.add(bucket.path("key").asText()); + } + return keys; + } + + private JsonNode runSearch(String index, String body) throws Exception { + try (var response = + openSearchClient + .generic() + .execute( + Requests.builder() + .method("POST") + .endpoint("/" + index + "/_search") + .json(body) + .build())) { + String raw = response.getBody().map(b -> b.bodyAsString()).orElse("{}"); + return mapper.readTree(raw); + } + } + + private void createIndex(String index, String mappingResource) throws Exception { + String rawMapping; + try (InputStream in = getClass().getResourceAsStream(mappingResource)) { + assertNotNull(in, "Mapping resource not found on classpath: " + mappingResource); + rawMapping = new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + String enriched = stripLanguageAnalysis(OsUtils.enrichIndexMappingForOpenSearch(rawMapping)); + try (var response = + openSearchClient + .generic() + .execute( + Requests.builder().method("PUT").endpoint("/" + index).json(enriched).build())) { + if (response.getStatus() >= 400) { + throw new IOException( + "Failed to create index " + + index + + ": " + + response.getBody().map(b -> b.bodyAsString()).orElse("no body")); + } + } + } + + private void indexDocument(String index, String id, String document) throws Exception { + try (var response = + openSearchClient + .generic() + .execute( + Requests.builder() + .method("PUT") + .endpoint("/" + index + "/_doc/" + id + "?refresh=true") + .json(document) + .build())) { + if (response.getStatus() >= 400) { + throw new IOException( + "Failed to index document into " + + index + + ": " + + response.getBody().map(b -> b.bodyAsString()).orElse("no body")); + } + } + } + + private String topicDocument() throws Exception { + Map document = + Map.ofEntries( + Map.entry("id", TOPIC_ID), + Map.entry("name", "orders"), + Map.entry("displayName", "Orders"), + Map.entry("fullyQualifiedName", TOPIC_FQN), + Map.entry("deleted", false), + Map.entry("entityType", "topic"), + Map.entry("fqnParts", List.of("svc", "ns", FQN_PART)), + Map.entry("tags", List.of(tagLabel(TAG_FQN))), + Map.entry("tier", tagLabel(TIER_FQN)), + Map.entry("certification", Map.of("tagLabel", Map.of("tagFQN", CERTIFICATION_FQN))), + Map.entry("owners", List.of(Map.of("id", OWNER_ID, "type", "user", "name", "alice"))), + Map.entry( + "domains", + List.of( + Map.of( + "id", + DOMAIN_ID, + "type", + "domain", + "name", + DOMAIN_FQN, + "fullyQualifiedName", + DOMAIN_FQN)))); + return mapper.writeValueAsString(document); + } + + private String testCaseDocument() throws Exception { + Map document = + Map.of( + "id", + TEST_CASE_ID, + "name", + "amount_not_null", + "fullyQualifiedName", + TEST_CASE_FQN, + "deleted", + false, + "entityType", + "testCase", + "testCaseResult", + Map.of("testCaseStatus", TEST_CASE_STATUS, "timestamp", 1700000000000L)); + return mapper.writeValueAsString(document); + } + + private Map tagLabel(String tagFqn) { + return Map.of( + "tagFQN", tagFqn, + "labelType", "Manual", + "source", "Classification", + "state", "Confirmed"); + } + + private String termQuery(String field, String value) throws Exception { + return mapper.writeValueAsString(Map.of("query", Map.of("term", Map.of(field, value)))); + } + + private String nestedTermQuery(String path, String field, String value) throws Exception { + return mapper.writeValueAsString( + Map.of( + "query", + Map.of("nested", Map.of("path", path, "query", Map.of("term", Map.of(field, value)))))); + } + + private String termsAggregation(String aggName, String field) throws Exception { + return mapper.writeValueAsString( + Map.of("size", 0, "aggs", Map.of(aggName, Map.of("terms", Map.of("field", field))))); + } + + private String topicIndex(String language) { + return "behavior_topic_" + language; + } + + private String testCaseIndex(String language) { + return "behavior_testcase_" + language; + } + + /** + * Remove every analysis setting and field-level {@code analyzer}/{@code search_analyzer}/{@code + * normalizer} reference so each language's index is creatable on a vanilla OpenSearch image. The + * denormalized consumer fields validated here are all {@code keyword}/{@code nested} and therefore + * analyzer-independent; per-language text analyzers (en stemmer, jp kuromoji, ...) are a separate + * concern, intentionally out of scope. What remains per language is the real field structure, so + * structural drift between languages — e.g. {@code jp/topic} dropping top-level {@code domains} — + * is still caught. + */ + private String stripLanguageAnalysis(String mappingJson) throws Exception { + JsonNode root = mapper.readTree(mappingJson); + if (root.path("settings") instanceof ObjectNode settings) { + settings.remove("analysis"); + if (settings.path("index") instanceof ObjectNode indexSettings) { + indexSettings.remove("analysis"); + } + } + stripAnalysisReferences(root.path("mappings")); + return mapper.writeValueAsString(root); + } + + private void stripAnalysisReferences(JsonNode node) { + if (node instanceof ObjectNode object) { + object.remove("analyzer"); + object.remove("search_analyzer"); + object.remove("normalizer"); + object.forEach(this::stripAnalysisReferences); + } else if (node.isArray()) { + node.forEach(this::stripAnalysisReferences); + } + } +} From 73282d01ad08fda6b7c7b0a47ec51183b26ff659 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Fri, 5 Jun 2026 15:13:03 +0200 Subject: [PATCH 03/19] fix(search): define missing jp index-mapping analyzers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 30 of 55 jp index mappings referenced an analyzer the file does not define: an incomplete om_analyzer <-> om_analyzer_jp split left field-level analyzer / search_analyzer references dangling, so those jp indexes fail to create ("analyzer [...] has not been configured") whenever searchIndexMappingLanguage is set to jp. It is invisible on en-default deployments, which never load these files — the same reason the jp/topic domains drop went unnoticed. Add the missing analyzer definition to each file, preserving every field's existing reference: 29 files gain the generic om_analyzer (letter + lowercase + om_stemmer, matching the other jp mappings) used by identifier fields, and api_collection gains the kuromoji om_analyzer_jp used by its text fields. No field reference is changed, so prose stays kuromoji-analyzed while identifier fields (tagFQN, column names, custom-property text) stay on the generic analyzer, consistent with en and the already-correct jp mappings. jp still requires the analysis-kuromoji plugin. Surfaced by SearchConsumerFieldBehaviorIT. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../elasticsearch/jp/api_collection_index_mapping.json | 10 ++++++++++ .../elasticsearch/jp/api_endpoint_index_mapping.json | 7 +++++++ .../elasticsearch/jp/api_service_index_mapping.json | 7 +++++++ .../elasticsearch/jp/chart_index_mapping.json | 7 +++++++ .../elasticsearch/jp/container_index_mapping.json | 7 +++++++ .../jp/dashboard_data_model_index_mapping.json | 7 +++++++ .../elasticsearch/jp/dashboard_index_mapping.json | 7 +++++++ .../jp/dashboard_service_index_mapping.json | 7 +++++++ .../jp/database_service_index_mapping.json | 7 +++++++ .../elasticsearch/jp/domain_index_mapping.json | 7 +++++++ .../elasticsearch/jp/drive_service_index_mapping.json | 7 +++++++ .../elasticsearch/jp/glossary_index_mapping.json | 7 +++++++ .../elasticsearch/jp/glossary_term_index_mapping.json | 7 +++++++ .../jp/ingestion_pipeline_index_mapping.json | 7 +++++++ .../jp/messaging_service_index_mapping.json | 7 +++++++ .../elasticsearch/jp/metric_index_mapping.json | 7 +++++++ .../elasticsearch/jp/mlmodel_index_mapping.json | 7 +++++++ .../jp/mlmodel_service_index_mapping.json | 7 +++++++ .../elasticsearch/jp/pipeline_index_mapping.json | 7 +++++++ .../jp/pipeline_service_index_mapping.json | 7 +++++++ .../jp/query_cost_record_index_mapping.json | 7 +++++++ .../elasticsearch/jp/query_index_mapping.json | 7 +++++++ .../elasticsearch/jp/search_entity_index_mapping.json | 7 +++++++ .../elasticsearch/jp/search_service_index_mapping.json | 7 +++++++ .../jp/security_service_index_mapping.json | 7 +++++++ .../jp/storage_service_index_mapping.json | 7 +++++++ .../jp/stored_procedure_index_mapping.json | 7 +++++++ .../jp/test_case_result_index_mapping.json | 7 +++++++ .../elasticsearch/jp/test_suite_index_mapping.json | 7 +++++++ .../elasticsearch/jp/topic_index_mapping.json | 7 +++++++ 30 files changed, 213 insertions(+) diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json index e125214ddc98..fdf4d4fda27e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json @@ -25,6 +25,16 @@ } }, "analyzer": { + "om_analyzer_jp": { + "type": "custom", + "tokenizer": "kuromoji_tokenizer", + "filter": [ + "kuromoji_baseform", + "kuromoji_part_of_speech", + "kuromoji_number", + "kuromoji_stemmer" + ] + }, "om_analyzer": { "tokenizer": "standard", "filter": [ diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json index e6e25cde8a52..7eac2057afc3 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json index ae015a6cfbf5..b13a63e9b373 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json index 8d30a8b09c3c..d3f367d6da81 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json index 47773e2b845d..df86816e4c8a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json index 9386a2da3f47..712c346ef257 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json @@ -14,6 +14,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index c6bbeefe036c..2d551b58cd85 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -14,6 +14,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json index e5eb26c5fdd9..c501aba1bb8b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json index 6630bfec621d..beea559c580e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json index 144848c8a900..5e751bb5511e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json @@ -14,6 +14,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json index b469241dab49..a9ca272cc183 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json index b9687fb48ef8..dab6a4908ee9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json @@ -14,6 +14,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json index 5e52a1f95c08..e55e3c877493 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json @@ -14,6 +14,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json index 6e8b2131e907..1011208f3d48 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json index 7c2ee7bd0c9c..8531d24df7c6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json index 082fa1201e6e..806e1cb8eb91 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index b98478bf0eea..93ac44078a56 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json index ffea41953286..89102b90d103 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json index ac62855eb46b..5f8eb4aa35d9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json index 98dd8de4aa57..c1583d2154cf 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/query_cost_record_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/query_cost_record_index_mapping.json index 227a5dda0b47..da974a232c38 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/query_cost_record_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/query_cost_record_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/query_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/query_index_mapping.json index 22c9bfbff33c..91a7c816cb75 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/query_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/query_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 31247a92daf9..5fec78205efa 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json index ae015a6cfbf5..b13a63e9b373 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json index 7097ca5cffe9..309bec3b4ac1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json index 665799638f0d..7a70561fc383 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json index 731bbed20021..c38c2b6cc5d9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json index c3c4796fa655..c808fe84aa93 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_result_index_mapping.json @@ -15,6 +15,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json index c033ef8c1692..d02cbd44b608 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json @@ -12,6 +12,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json index 198bae38bcc8..ed7173993235 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -11,6 +11,13 @@ } }, "analyzer": { + "om_analyzer": { + "tokenizer": "letter", + "filter": [ + "lowercase", + "om_stemmer" + ] + }, "om_analyzer_jp": { "tokenizer": "kuromoji_tokenizer", "type": "custom", From 7168e2fd708059e74f82559c967f82e8a4d5299c Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 12:47:26 +0200 Subject: [PATCH 04/19] test(search): install analysis-kuromoji in the IT OpenSearch image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Build the integration-test OpenSearch container from an image with the language analysis plugins installed (analysis-kuromoji for Japanese) via a shared SearchTestImages helper, wired into both TestSuiteBootstrap and ContainerizedServer. The plugins are installed unconditionally — an English-only run never references them — so the search IT suite can be run for non-English mapping languages. That is what would have caught the jp analyzer drift earlier: CI only ever ran en on a vanilla image, where the jp mappings (kuromoji) could never even be created. Add OpenSearchLanguageAnalyzerIT: boots the plugin image and creates the real en/ru/jp topic index mappings with their own analyzers (no analysis stripping), asserting analysis-kuromoji is installed and that the jp mappings — which reference kuromoji and previously had dangling analyzer references — now create cleanly. zh (third-party analysis-ik) is intentionally out of scope. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../it/bootstrap/TestSuiteBootstrap.java | 4 +- .../it/server/ContainerizedServer.java | 3 +- .../it/server/SearchTestImages.java | 45 +++++++ .../tests/OpenSearchLanguageAnalyzerIT.java | 118 ++++++++++++++++++ 4 files changed, 168 insertions(+), 2 deletions(-) create mode 100644 openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java create mode 100644 openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java index 0c90c0053ac0..5ada3bac86d1 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/bootstrap/TestSuiteBootstrap.java @@ -42,6 +42,7 @@ import org.jdbi.v3.sqlobject.SqlObjects; import org.junit.platform.launcher.LauncherSession; import org.junit.platform.launcher.LauncherSessionListener; +import org.openmetadata.it.server.SearchTestImages; import org.openmetadata.it.util.SdkClients; import org.openmetadata.schema.api.configuration.pipelineServiceClient.Parameters; import org.openmetadata.schema.api.configuration.pipelineServiceClient.PipelineServiceClientConfiguration; @@ -350,7 +351,8 @@ private void startSearch() { LOG.info("Starting OpenSearch container with image: {}", image); org.opensearch.testcontainers.OpensearchContainer opensearch = - new org.opensearch.testcontainers.OpensearchContainer<>(image); + new org.opensearch.testcontainers.OpensearchContainer<>( + SearchTestImages.openSearchWithAnalysisPlugins(image)); opensearch.withEnv("discovery.type", "single-node"); opensearch.withEnv("DISABLE_SECURITY_PLUGIN", "true"); opensearch.withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true"); diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/ContainerizedServer.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/ContainerizedServer.java index ae8eb888189d..6054be132abd 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/ContainerizedServer.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/ContainerizedServer.java @@ -224,7 +224,8 @@ private static MySQLContainer newMysql(final Network network) { } private static OpensearchContainer newOpenSearch(final Network network) { - return new OpensearchContainer<>("opensearchproject/opensearch:3.4.0") + return new OpensearchContainer<>( + SearchTestImages.openSearchWithAnalysisPlugins("opensearchproject/opensearch:3.4.0")) .withNetwork(network) .withNetworkAliases(SEARCH_ALIAS) .withEnv("discovery.type", "single-node") diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java new file mode 100644 index 000000000000..36ebe1eba635 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java @@ -0,0 +1,45 @@ +package org.openmetadata.it.server; + +import org.testcontainers.images.builder.ImageFromDockerfile; +import org.testcontainers.utility.DockerImageName; + +/** + * Builds the OpenSearch test image with the language-analysis plugins baked in, so the integration + * tests can exercise non-English mapping languages (e.g. {@code analysis-kuromoji} for Japanese). + * + *

The plugins are installed unconditionally: a run that only uses the English mappings never + * references them, while a run configured for {@code jp} can create indexes whose text fields use + * {@code kuromoji_tokenizer}. This is what lets the search IT suite catch per-language + * mapping/analyzer drift — the jp mappings referencing undefined analyzers went unnoticed precisely + * because CI only ever ran English on a vanilla image. + * + *

Chinese ({@code zh}) uses the third-party IK plugin ({@code analysis-ik}); it is intentionally + * not installed here because it ships only as a version-matched release URL that would break the + * image build if it became unavailable. Add it separately when zh coverage is needed. + */ +public final class SearchTestImages { + + private static final String OPENSEARCH_BASE_REFERENCE = "opensearchproject/opensearch"; + private static final String ANALYSIS_PLUGINS = "analysis-kuromoji"; + + private SearchTestImages() {} + + /** + * Returns a {@link DockerImageName} for {@code baseImage} with the analysis plugins installed. The + * image is built once per run and reused via Docker's layer cache. + */ + public static DockerImageName openSearchWithAnalysisPlugins(String baseImage) { + String builtImage = + new ImageFromDockerfile() + .withDockerfileFromBuilder( + builder -> + builder + .from(baseImage) + .run( + "/usr/share/opensearch/bin/opensearch-plugin install --batch " + + ANALYSIS_PLUGINS) + .build()) + .get(); + return DockerImageName.parse(builtImage).asCompatibleSubstituteFor(OPENSEARCH_BASE_REFERENCE); + } +} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java new file mode 100644 index 000000000000..11cd34d666b4 --- /dev/null +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java @@ -0,0 +1,118 @@ +package org.openmetadata.it.tests; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import org.apache.hc.core5.http.HttpHost; +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.openmetadata.it.server.SearchTestImages; +import org.openmetadata.service.search.opensearch.OsUtils; +import org.opensearch.testcontainers.OpensearchContainer; +import org.testcontainers.junit.jupiter.Container; +import org.testcontainers.junit.jupiter.Testcontainers; +import os.org.opensearch.client.json.jackson.JacksonJsonpMapper; +import os.org.opensearch.client.opensearch.OpenSearchClient; +import os.org.opensearch.client.opensearch.generic.Requests; +import os.org.opensearch.client.transport.httpclient5.ApacheHttpClient5Transport; +import os.org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; + +/** + * Verifies that the integration-test OpenSearch image ships the language-analysis plugins (see + * {@link SearchTestImages}) and that the real, non-English index mappings — which reference those + * plugins — create cleanly with their own analyzers (no analysis stripping). + * + *

This is the end-to-end guard the search suite lacked: jp text fields use {@code + * kuromoji_tokenizer}, so a vanilla image could never create the jp indexes, and the jp mappings' + * analyzer drift (fields referencing analyzers the file did not define) was invisible until now. + * Creating the real jp mapping here fails if either the plugin is missing or an analyzer reference + * is dangling. + * + *

en and ru use only built-in analyzers; jp uses {@code analysis-kuromoji}. zh ({@code + * analysis-ik}) is excluded because that third-party plugin is not installed. + */ +@Testcontainers +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +class OpenSearchLanguageAnalyzerIT { + + private static final List LANGUAGES_WITH_AVAILABLE_ANALYZERS = List.of("en", "ru", "jp"); + + @Container + static OpensearchContainer opensearch = + new OpensearchContainer<>( + SearchTestImages.openSearchWithAnalysisPlugins("opensearchproject/opensearch:3.4.0")) + .withStartupTimeout(Duration.ofMinutes(5)) + .withEnv("discovery.type", "single-node") + .withEnv("DISABLE_SECURITY_PLUGIN", "true") + .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") + .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m"); + + private OpenSearchClient openSearchClient; + + @BeforeAll + void setUp() { + HttpHost httpHost = new HttpHost("http", opensearch.getHost(), opensearch.getMappedPort(9200)); + ApacheHttpClient5Transport transport = + ApacheHttpClient5TransportBuilder.builder(httpHost) + .setMapper(new JacksonJsonpMapper()) + .build(); + openSearchClient = new OpenSearchClient(transport); + } + + @AfterAll + void tearDown() throws Exception { + if (openSearchClient != null) { + openSearchClient._transport().close(); + } + } + + @Test + void kuromojiAnalysisPluginIsInstalled() throws Exception { + String plugins = execute("GET", "/_cat/plugins", null); + assertTrue( + plugins.contains("analysis-kuromoji"), + "The integration-test OpenSearch image must ship analysis-kuromoji. Installed: " + plugins); + } + + @Test + void realLanguageMappingsCreateWithTheirOwnAnalyzers() throws Exception { + List failures = new ArrayList<>(); + for (String language : LANGUAGES_WITH_AVAILABLE_ANALYZERS) { + String mapping = + OsUtils.enrichIndexMappingForOpenSearch( + readResource("/elasticsearch/" + language + "/topic_index_mapping.json")); + String response = execute("PUT", "/lang_topic_" + language, mapping); + if (!response.contains("\"acknowledged\":true")) { + failures.add(language + " -> " + response); + } + } + assertTrue( + failures.isEmpty(), + "Real index mapping (analyzers included) failed to create for language(s): " + failures); + } + + private String readResource(String path) throws IOException { + try (InputStream in = getClass().getResourceAsStream(path)) { + assertNotNull(in, "Mapping resource not found on classpath: " + path); + return new String(in.readAllBytes(), StandardCharsets.UTF_8); + } + } + + private String execute(String method, String endpoint, String body) throws Exception { + var builder = Requests.builder().method(method).endpoint(endpoint); + if (body != null) { + builder.json(body); + } + try (var response = openSearchClient.generic().execute(builder.build())) { + return response.getBody().map(b -> b.bodyAsString()).orElse(""); + } + } +} From cd510891530cc2b0eb45723d9f9caffcf32e09c5 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 13:09:04 +0200 Subject: [PATCH 05/19] test(search): validate all languages with real analyzers in one search IT MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Run the per-language consumer-contract checks directly in the search IT, so the whole integration suite does not need to run once per language. The container now uses the analysis-plugin image (SearchTestImages), so en/ru/jp index with their real analyzers (jp = kuromoji) — which also exercises jp analyzer resolution end to end — and only zh (third-party analysis-ik, not installed) is validated structurally with its analysis stripped. Fold the plugin-present and real-jp-index creation checks into this IT and drop the standalone OpenSearchLanguageAnalyzerIT. Aggregation assertions now compare bucket keys case-insensitively: real keyword fields carry lowercase_normalizer (testCaseResult.testCaseStatus in every language, tier.tagFQN in en only), so the bucket key is lowercased while the feature still returns the asset. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/OpenSearchLanguageAnalyzerIT.java | 118 ------------------ .../tests/SearchConsumerFieldBehaviorIT.java | 111 ++++++++++++---- 2 files changed, 84 insertions(+), 145 deletions(-) delete mode 100644 openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java deleted file mode 100644 index 11cd34d666b4..000000000000 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/OpenSearchLanguageAnalyzerIT.java +++ /dev/null @@ -1,118 +0,0 @@ -package org.openmetadata.it.tests; - -import static org.junit.jupiter.api.Assertions.assertNotNull; -import static org.junit.jupiter.api.Assertions.assertTrue; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.ArrayList; -import java.util.List; -import org.apache.hc.core5.http.HttpHost; -import org.junit.jupiter.api.AfterAll; -import org.junit.jupiter.api.BeforeAll; -import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.TestInstance; -import org.openmetadata.it.server.SearchTestImages; -import org.openmetadata.service.search.opensearch.OsUtils; -import org.opensearch.testcontainers.OpensearchContainer; -import org.testcontainers.junit.jupiter.Container; -import org.testcontainers.junit.jupiter.Testcontainers; -import os.org.opensearch.client.json.jackson.JacksonJsonpMapper; -import os.org.opensearch.client.opensearch.OpenSearchClient; -import os.org.opensearch.client.opensearch.generic.Requests; -import os.org.opensearch.client.transport.httpclient5.ApacheHttpClient5Transport; -import os.org.opensearch.client.transport.httpclient5.ApacheHttpClient5TransportBuilder; - -/** - * Verifies that the integration-test OpenSearch image ships the language-analysis plugins (see - * {@link SearchTestImages}) and that the real, non-English index mappings — which reference those - * plugins — create cleanly with their own analyzers (no analysis stripping). - * - *

This is the end-to-end guard the search suite lacked: jp text fields use {@code - * kuromoji_tokenizer}, so a vanilla image could never create the jp indexes, and the jp mappings' - * analyzer drift (fields referencing analyzers the file did not define) was invisible until now. - * Creating the real jp mapping here fails if either the plugin is missing or an analyzer reference - * is dangling. - * - *

en and ru use only built-in analyzers; jp uses {@code analysis-kuromoji}. zh ({@code - * analysis-ik}) is excluded because that third-party plugin is not installed. - */ -@Testcontainers -@TestInstance(TestInstance.Lifecycle.PER_CLASS) -class OpenSearchLanguageAnalyzerIT { - - private static final List LANGUAGES_WITH_AVAILABLE_ANALYZERS = List.of("en", "ru", "jp"); - - @Container - static OpensearchContainer opensearch = - new OpensearchContainer<>( - SearchTestImages.openSearchWithAnalysisPlugins("opensearchproject/opensearch:3.4.0")) - .withStartupTimeout(Duration.ofMinutes(5)) - .withEnv("discovery.type", "single-node") - .withEnv("DISABLE_SECURITY_PLUGIN", "true") - .withEnv("DISABLE_INSTALL_DEMO_CONFIG", "true") - .withEnv("OPENSEARCH_JAVA_OPTS", "-Xms512m -Xmx512m"); - - private OpenSearchClient openSearchClient; - - @BeforeAll - void setUp() { - HttpHost httpHost = new HttpHost("http", opensearch.getHost(), opensearch.getMappedPort(9200)); - ApacheHttpClient5Transport transport = - ApacheHttpClient5TransportBuilder.builder(httpHost) - .setMapper(new JacksonJsonpMapper()) - .build(); - openSearchClient = new OpenSearchClient(transport); - } - - @AfterAll - void tearDown() throws Exception { - if (openSearchClient != null) { - openSearchClient._transport().close(); - } - } - - @Test - void kuromojiAnalysisPluginIsInstalled() throws Exception { - String plugins = execute("GET", "/_cat/plugins", null); - assertTrue( - plugins.contains("analysis-kuromoji"), - "The integration-test OpenSearch image must ship analysis-kuromoji. Installed: " + plugins); - } - - @Test - void realLanguageMappingsCreateWithTheirOwnAnalyzers() throws Exception { - List failures = new ArrayList<>(); - for (String language : LANGUAGES_WITH_AVAILABLE_ANALYZERS) { - String mapping = - OsUtils.enrichIndexMappingForOpenSearch( - readResource("/elasticsearch/" + language + "/topic_index_mapping.json")); - String response = execute("PUT", "/lang_topic_" + language, mapping); - if (!response.contains("\"acknowledged\":true")) { - failures.add(language + " -> " + response); - } - } - assertTrue( - failures.isEmpty(), - "Real index mapping (analyzers included) failed to create for language(s): " + failures); - } - - private String readResource(String path) throws IOException { - try (InputStream in = getClass().getResourceAsStream(path)) { - assertNotNull(in, "Mapping resource not found on classpath: " + path); - return new String(in.readAllBytes(), StandardCharsets.UTF_8); - } - } - - private String execute(String method, String endpoint, String body) throws Exception { - var builder = Requests.builder().method(method).endpoint(endpoint); - if (body != null) { - builder.json(body); - } - try (var response = openSearchClient.generic().execute(builder.build())) { - return response.getBody().map(b -> b.bodyAsString()).orElse(""); - } - } -} diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java index bcc1c116e4c2..c1c3c7e2e22d 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java @@ -18,11 +18,11 @@ import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; +import org.openmetadata.it.server.SearchTestImages; import org.openmetadata.service.search.opensearch.OsUtils; import org.opensearch.testcontainers.OpensearchContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; -import org.testcontainers.utility.DockerImageName; import os.org.opensearch.client.json.jackson.JacksonJsonpMapper; import os.org.opensearch.client.opensearch.OpenSearchClient; import os.org.opensearch.client.opensearch.generic.Requests; @@ -37,15 +37,21 @@ * reported as a broken feature in a specific language (e.g. "Domain filter is broken in * [jp]"), which is what an operator can act on — not "index is missing a mapping". * - *

This is the test that would have caught the {@code jp/topic} mapping dropping top-level {@code - * domains}: the domain-filter assertion below returns zero hits for {@code jp} until the mapping is - * fixed. The earlier structural unit test could not, because it only knew which fields to look for - * from a hand-maintained list. + *

This is the search-side coverage the suite lacked for non-English languages — without the need + * to run the whole integration-test suite once per language. The container is built with the + * language-analysis plugins ({@link SearchTestImages}), so en, ru and jp index with their + * real analyzers (jp uses {@code kuromoji_tokenizer}); creating the real jp mapping here + * therefore also catches analyzer drift (fields referencing analyzers the mapping never defined). + * zh uses the third-party {@code analysis-ik} plugin, which is not installed, so zh alone has its + * analysis stripped and is validated structurally — the consumer fields are {@code keyword}/{@code + * nested} and analyzer-independent, so every assertion still holds. * - *

The query types mirror real consumers: a nested query for {@code owners} (RBAC isOwner), term - * filters for tags/tier/certification/domains (Explore facets, RBAC, Data Quality), a terms - * aggregation for {@code testCaseResult.testCaseStatus} (the Data Quality execution summary), and a - * keyword term on {@code fqnParts} (hierarchical search / autocomplete). + *

It would have caught the {@code jp/topic} mapping dropping top-level {@code domains} (the + * domain-filter assertion returns zero hits for {@code jp}) and the jp analyzer references that made + * jp indexes uncreatable. The query types mirror real consumers: a nested query for {@code owners} + * (RBAC isOwner), term filters for tags/tier/certification/domains (Explore facets, RBAC, Data + * Quality), a terms aggregation for {@code testCaseResult.testCaseStatus} (the Data Quality + * execution summary), and a keyword term on {@code fqnParts} (hierarchical search / autocomplete). */ @Testcontainers @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -68,7 +74,8 @@ class SearchConsumerFieldBehaviorIT { @Container static OpensearchContainer opensearch = - new OpensearchContainer<>(DockerImageName.parse("opensearchproject/opensearch:3.4.0")) + new OpensearchContainer<>( + SearchTestImages.openSearchWithAnalysisPlugins("opensearchproject/opensearch:3.4.0")) .withStartupTimeout(Duration.ofMinutes(5)) .withEnv("discovery.type", "single-node") .withEnv("OPENSEARCH_INITIAL_ADMIN_PASSWORD", "Test@12345") @@ -90,10 +97,16 @@ void setUp() throws Exception { mapper = new ObjectMapper(); for (String language : LANGUAGES) { - createIndex(topicIndex(language), "/elasticsearch/" + language + "/topic_index_mapping.json"); + boolean stripAnalysis = needsAnalysisStrip(language); + createIndex( + topicIndex(language), + "/elasticsearch/" + language + "/topic_index_mapping.json", + stripAnalysis); indexDocument(topicIndex(language), TOPIC_ID, topicDocument()); createIndex( - testCaseIndex(language), "/elasticsearch/" + language + "/test_case_index_mapping.json"); + testCaseIndex(language), + "/elasticsearch/" + language + "/test_case_index_mapping.json", + stripAnalysis); indexDocument(testCaseIndex(language), TEST_CASE_ID, testCaseDocument()); } } @@ -117,8 +130,8 @@ void tierAggregationHasBucketInAllLanguages() throws Exception { assertFeatureWorksInAllLanguages( "Tier aggregation (Explore tier facet, Data Quality tier widget)", language -> - bucketKeys(topicIndex(language), termsAggregation("tier", "tier.tagFQN"), "tier") - .contains(TIER_FQN)); + aggregationHasBucket( + topicIndex(language), termsAggregation("tier", "tier.tagFQN"), "tier", TIER_FQN)); } @Test @@ -160,11 +173,26 @@ void dataQualityStatusAggregationHasBucketInAllLanguages() throws Exception { assertFeatureWorksInAllLanguages( "Data Quality status aggregation (Test Suite execution summary, DQ status filter)", language -> - bucketKeys( - testCaseIndex(language), - termsAggregation("status", "testCaseResult.testCaseStatus"), - "status") - .contains(TEST_CASE_STATUS)); + aggregationHasBucket( + testCaseIndex(language), + termsAggregation("status", "testCaseResult.testCaseStatus"), + "status", + TEST_CASE_STATUS)); + } + + @Test + void japaneseAnalysisPluginIsInstalled() throws Exception { + try (var response = + openSearchClient + .generic() + .execute(Requests.builder().method("GET").endpoint("/_cat/plugins").build())) { + String plugins = response.getBody().map(b -> b.bodyAsString()).orElse(""); + assertTrue( + plugins.contains("analysis-kuromoji"), + "The integration-test OpenSearch image must ship analysis-kuromoji so jp mappings index " + + "with their real kuromoji analyzer instead of a stripped fallback. Installed: " + + plugins); + } } @FunctionalInterface @@ -194,6 +222,23 @@ private int hits(String index, String queryBody) throws Exception { return runSearch(index, queryBody).path("hits").path("hits").size(); } + /** + * Whether an aggregation produced a bucket for {@code expected}, compared case-insensitively. A + * keyword field may carry {@code lowercase_normalizer} (e.g. {@code testCaseResult.testCaseStatus} + * in every language, {@code tier.tagFQN} in en only), which lowercases the stored value and hence + * the bucket key; the feature still works — a bucket exists for our asset — so either case passes. + * A missing field or value produces no bucket at all and still fails the assertion. + */ + private boolean aggregationHasBucket( + String index, String aggBody, String aggName, String expected) throws Exception { + for (String key : bucketKeys(index, aggBody, aggName)) { + if (key.equalsIgnoreCase(expected)) { + return true; + } + } + return false; + } + private List bucketKeys(String index, String aggBody, String aggName) throws Exception { List keys = new ArrayList<>(); JsonNode buckets = runSearch(index, aggBody).path("aggregations").path(aggName).path("buckets"); @@ -218,13 +263,17 @@ private JsonNode runSearch(String index, String body) throws Exception { } } - private void createIndex(String index, String mappingResource) throws Exception { + private void createIndex(String index, String mappingResource, boolean stripAnalysis) + throws Exception { String rawMapping; try (InputStream in = getClass().getResourceAsStream(mappingResource)) { assertNotNull(in, "Mapping resource not found on classpath: " + mappingResource); rawMapping = new String(in.readAllBytes(), StandardCharsets.UTF_8); } - String enriched = stripLanguageAnalysis(OsUtils.enrichIndexMappingForOpenSearch(rawMapping)); + String enriched = OsUtils.enrichIndexMappingForOpenSearch(rawMapping); + if (stripAnalysis) { + enriched = stripLanguageAnalysis(enriched); + } try (var response = openSearchClient .generic() @@ -339,14 +388,22 @@ private String testCaseIndex(String language) { return "behavior_testcase_" + language; } + /** + * Languages whose analyzers are available in the IT image keep their real analysis (en/ru are + * built-in, jp uses the installed {@code analysis-kuromoji}). Only zh — which needs the + * uninstalled third-party {@code analysis-ik} — has its analysis stripped, so its index is still + * creatable for structural validation. The consumer fields asserted here are {@code keyword}/{@code + * nested} and analyzer-independent, so stripping zh does not weaken any assertion. + */ + private boolean needsAnalysisStrip(String language) { + return "zh".equals(language); + } + /** * Remove every analysis setting and field-level {@code analyzer}/{@code search_analyzer}/{@code - * normalizer} reference so each language's index is creatable on a vanilla OpenSearch image. The - * denormalized consumer fields validated here are all {@code keyword}/{@code nested} and therefore - * analyzer-independent; per-language text analyzers (en stemmer, jp kuromoji, ...) are a separate - * concern, intentionally out of scope. What remains per language is the real field structure, so - * structural drift between languages — e.g. {@code jp/topic} dropping top-level {@code domains} — - * is still caught. + * normalizer} reference so an index is creatable without its language-analysis plugin (used only + * for zh). What remains is the real field structure, so structural drift — e.g. a language mapping + * dropping a top-level field — is still caught. */ private String stripLanguageAnalysis(String mappingJson) throws Exception { JsonNode root = mapper.readTree(mappingJson); From ebd2ffc664bc8168bf97aba6b5e5eed8534d8f72 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 14:38:38 +0200 Subject: [PATCH 06/19] fix(search): align jp/ru/zh keyword normalizers to en (case-insensitive) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit en uses lowercase_normalizer on 45 keyword field paths (tier.tagFQN, displayName.keyword, owners.name, dataProducts.*, ...) so those fields filter, sort and aggregate case-insensitively. jp/ru/zh were missing the normalizer on those fields, so the same Explore facets, RBAC rules and Data Quality widgets behaved case-sensitively — and inconsistently with en — for non-English deployments. Add lowercase_normalizer to the matching keyword fields in jp/ru/zh wherever en has it (216 occurrences across 122 files). The normalizer is already defined in every file, so this only adds references. Surfaced by SearchConsumerFieldBehaviorIT (tier aggregation diverged en vs jp/ru). Side effects: 3 files (jp/zh test_case, jp test_suite) had a duplicate settings.index key collapsed. 57 further en-only normalizers could not be aligned because the field is missing or non-keyword in the other language — a separate structural drift left for follow-up. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../jp/api_collection_index_mapping.json | 3 +- .../jp/api_endpoint_index_mapping.json | 3 +- .../jp/api_service_index_mapping.json | 3 +- .../elasticsearch/jp/chart_index_mapping.json | 6 ++-- .../jp/container_index_mapping.json | 3 +- .../dashboard_data_model_index_mapping.json | 9 +++-- .../jp/dashboard_index_mapping.json | 6 ++-- .../jp/dashboard_service_index_mapping.json | 3 +- .../jp/data_products_index_mapping.json | 3 +- .../jp/database_index_mapping.json | 3 +- .../jp/database_schema_index_mapping.json | 3 +- .../jp/database_service_index_mapping.json | 3 +- .../jp/directory_index_mapping.json | 6 ++-- .../jp/domain_index_mapping.json | 3 +- .../jp/drive_service_index_mapping.json | 3 +- .../elasticsearch/jp/file_index_mapping.json | 6 ++-- .../jp/glossary_index_mapping.json | 3 +- .../jp/glossary_term_index_mapping.json | 6 ++-- .../jp/ingestion_pipeline_index_mapping.json | 9 +++-- .../jp/messaging_service_index_mapping.json | 3 +- .../jp/metadata_service_index_mapping.json | 3 +- .../jp/metric_index_mapping.json | 3 +- .../jp/mlmodel_index_mapping.json | 3 +- .../jp/mlmodel_service_index_mapping.json | 3 +- .../jp/pipeline_index_mapping.json | 6 ++-- .../jp/pipeline_service_index_mapping.json | 3 +- .../jp/search_entity_index_mapping.json | 3 +- .../jp/search_service_index_mapping.json | 3 +- .../jp/security_service_index_mapping.json | 18 ++++++---- .../jp/spreadsheet_index_mapping.json | 6 ++-- .../jp/storage_service_index_mapping.json | 3 +- .../jp/stored_procedure_index_mapping.json | 3 +- .../elasticsearch/jp/table_index_mapping.json | 6 ++-- .../elasticsearch/jp/tag_index_mapping.json | 3 +- .../elasticsearch/jp/team_index_mapping.json | 3 +- .../jp/test_case_index_mapping.json | 10 +++--- ..._case_resolution_status_index_mapping.json | 6 ++-- .../jp/test_suite_index_mapping.json | 10 +++--- .../elasticsearch/jp/topic_index_mapping.json | 3 +- .../elasticsearch/jp/user_index_mapping.json | 3 +- .../jp/worksheet_index_mapping.json | 6 ++-- .../ru/api_collection_index_mapping.json | 6 ++-- .../ru/api_endpoint_index_mapping.json | 6 ++-- .../ru/api_service_index_mapping.json | 6 ++-- .../elasticsearch/ru/chart_index_mapping.json | 6 ++-- .../ru/container_index_mapping.json | 3 +- .../dashboard_data_model_index_mapping.json | 6 ++-- .../ru/dashboard_index_mapping.json | 6 ++-- .../ru/dashboard_service_index_mapping.json | 6 ++-- .../ru/data_products_index_mapping.json | 3 +- .../ru/database_index_mapping.json | 6 ++-- .../ru/database_schema_index_mapping.json | 6 ++-- .../ru/database_service_index_mapping.json | 6 ++-- .../ru/directory_index_mapping.json | 30 +++++++++++------ .../ru/domain_index_mapping.json | 3 +- .../ru/drive_service_index_mapping.json | 3 +- .../elasticsearch/ru/file_index_mapping.json | 30 +++++++++++------ .../ru/glossary_index_mapping.json | 3 +- .../ru/glossary_term_index_mapping.json | 3 +- .../ru/ingestion_pipeline_index_mapping.json | 3 +- .../ru/messaging_service_index_mapping.json | 6 ++-- .../ru/metadata_service_index_mapping.json | 3 +- .../ru/metric_index_mapping.json | 3 +- .../ru/mlmodel_index_mapping.json | 6 ++-- .../ru/mlmodel_service_index_mapping.json | 6 ++-- .../ru/pipeline_index_mapping.json | 6 ++-- .../ru/pipeline_service_index_mapping.json | 6 ++-- .../ru/search_entity_index_mapping.json | 6 ++-- .../ru/search_service_index_mapping.json | 6 ++-- .../ru/security_service_index_mapping.json | 3 +- .../ru/spreadsheet_index_mapping.json | 30 +++++++++++------ .../ru/storage_service_index_mapping.json | 6 ++-- .../ru/stored_procedure_index_mapping.json | 6 ++-- .../elasticsearch/ru/table_index_mapping.json | 6 ++-- .../ru/test_case_index_mapping.json | 3 +- ..._case_resolution_status_index_mapping.json | 3 +- .../ru/test_case_result_index_mapping.json | 3 +- .../ru/test_suite_index_mapping.json | 3 +- .../elasticsearch/ru/topic_index_mapping.json | 6 ++-- .../ru/worksheet_index_mapping.json | 33 ++++++++++++------- .../zh/api_collection_index_mapping.json | 3 +- .../zh/api_endpoint_index_mapping.json | 3 +- .../zh/api_service_index_mapping.json | 3 +- .../elasticsearch/zh/chart_index_mapping.json | 9 +++-- .../zh/container_index_mapping.json | 3 +- .../dashboard_data_model_index_mapping.json | 6 ++-- .../zh/dashboard_index_mapping.json | 3 +- .../zh/dashboard_service_index_mapping.json | 3 +- .../zh/data_products_index_mapping.json | 3 +- .../zh/database_index_mapping.json | 3 +- .../zh/database_schema_index_mapping.json | 6 ++-- .../zh/database_service_index_mapping.json | 3 +- .../zh/directory_index_mapping.json | 6 ++-- .../zh/domain_index_mapping.json | 3 +- .../zh/drive_service_index_mapping.json | 3 +- .../elasticsearch/zh/file_index_mapping.json | 6 ++-- .../zh/glossary_index_mapping.json | 3 +- .../zh/glossary_term_index_mapping.json | 6 ++-- .../zh/ingestion_pipeline_index_mapping.json | 6 ++-- .../zh/messaging_service_index_mapping.json | 3 +- .../zh/metadata_service_index_mapping.json | 3 +- .../zh/metric_index_mapping.json | 3 +- .../zh/mlmodel_index_mapping.json | 3 +- .../zh/mlmodel_service_index_mapping.json | 3 +- .../zh/pipeline_index_mapping.json | 3 +- .../zh/pipeline_service_index_mapping.json | 3 +- .../zh/search_entity_index_mapping.json | 6 ++-- .../zh/search_service_index_mapping.json | 3 +- .../zh/security_service_index_mapping.json | 18 ++++++---- .../zh/spreadsheet_index_mapping.json | 6 ++-- .../zh/storage_service_index_mapping.json | 3 +- .../zh/stored_procedure_index_mapping.json | 3 +- .../elasticsearch/zh/table_index_mapping.json | 6 ++-- .../elasticsearch/zh/tag_index_mapping.json | 3 +- .../elasticsearch/zh/team_index_mapping.json | 3 +- .../zh/test_case_index_mapping.json | 10 +++--- ..._case_resolution_status_index_mapping.json | 3 +- .../zh/test_case_result_index_mapping.json | 3 +- .../zh/test_suite_index_mapping.json | 3 +- .../elasticsearch/zh/topic_index_mapping.json | 3 +- .../elasticsearch/zh/user_index_mapping.json | 3 +- .../zh/worksheet_index_mapping.json | 6 ++-- 122 files changed, 441 insertions(+), 228 deletions(-) diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json index fdf4d4fda27e..381f461a8103 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_collection_index_mapping.json @@ -162,7 +162,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json index 7eac2057afc3..c26670395e01 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_endpoint_index_mapping.json @@ -552,7 +552,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json index b13a63e9b373..efac4df758a7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/api_service_index_mapping.json @@ -203,7 +203,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json index d3f367d6da81..4e033e6b99df 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/chart_index_mapping.json @@ -307,7 +307,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -379,7 +380,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json index df86816e4c8a..4bb708fc119c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/container_index_mapping.json @@ -683,7 +683,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json index 712c346ef257..6f4e95693fa2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_data_model_index_mapping.json @@ -388,7 +388,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -504,7 +505,8 @@ "normalizer": "lowercase_normalizer" }, "dataModelType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "columns": { "properties": { @@ -574,7 +576,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json index 2d551b58cd85..6bb820af0032 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_index_mapping.json @@ -444,7 +444,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -623,7 +624,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json index c501aba1bb8b..341d03f76880 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/dashboard_service_index_mapping.json @@ -201,7 +201,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json index 8530ac688c0d..8736d528b67c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/data_products_index_mapping.json @@ -490,7 +490,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json index 4e8543cfc103..05ac8de4ca21 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_index_mapping.json @@ -158,7 +158,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json index cbf44c10cd80..01fe29b83e96 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_schema_index_mapping.json @@ -485,7 +485,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json index beea559c580e..53f0fb4ece0f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/database_service_index_mapping.json @@ -203,7 +203,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json index 9a377c7b800a..16d1c914d4b6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/directory_index_mapping.json @@ -192,7 +192,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -467,7 +468,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json index 5e751bb5511e..11f630ca222d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/domain_index_mapping.json @@ -241,7 +241,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json index a9ca272cc183..eb64a1d10dc2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/drive_service_index_mapping.json @@ -198,7 +198,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json index e3d1746cd9ca..db954269dc3f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/file_index_mapping.json @@ -207,7 +207,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -482,7 +483,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json index dab6a4908ee9..6a8ea9f5b6da 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_index_mapping.json @@ -312,7 +312,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json index e55e3c877493..12c9f33db446 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/glossary_term_index_mapping.json @@ -290,7 +290,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -463,7 +464,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json index 1011208f3d48..08802496e153 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/ingestion_pipeline_index_mapping.json @@ -267,7 +267,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -314,7 +315,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -365,7 +367,8 @@ } }, "pipelineType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "pipelineStatuses": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json index 8531d24df7c6..be7a64981a73 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/messaging_service_index_mapping.json @@ -202,7 +202,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/metadata_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/metadata_service_index_mapping.json index 7a612ff31b91..60647d81dd9c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/metadata_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/metadata_service_index_mapping.json @@ -200,7 +200,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json index 806e1cb8eb91..261ff300f3a6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/metric_index_mapping.json @@ -430,7 +430,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json index 93ac44078a56..41d2669b01c9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_index_mapping.json @@ -629,7 +629,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json index 89102b90d103..80968a1daac4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/mlmodel_service_index_mapping.json @@ -213,7 +213,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json index 5f8eb4aa35d9..0b4133b35bb2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_index_mapping.json @@ -379,7 +379,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -513,7 +514,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json index c1583d2154cf..9956e699d570 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/pipeline_service_index_mapping.json @@ -164,7 +164,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json index 5fec78205efa..ee486a66dbac 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_entity_index_mapping.json @@ -157,7 +157,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json index b13a63e9b373..efac4df758a7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/search_service_index_mapping.json @@ -203,7 +203,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json index 309bec3b4ac1..334ede20a22d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/security_service_index_mapping.json @@ -107,7 +107,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -356,7 +357,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -405,7 +407,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -430,7 +433,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -456,7 +460,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -596,7 +601,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json index da38cc093cde..2e2f991eba37 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/spreadsheet_index_mapping.json @@ -192,7 +192,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -467,7 +468,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json index 7a70561fc383..a6ec278145b9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/storage_service_index_mapping.json @@ -198,7 +198,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json index c38c2b6cc5d9..7b7200e7bd64 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/stored_procedure_index_mapping.json @@ -566,7 +566,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json index 3ce2e4559271..bbee140a1edd 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/table_index_mapping.json @@ -168,7 +168,8 @@ "type": "keyword" }, "tableType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "columns": { "properties": { @@ -856,7 +857,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/tag_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/tag_index_mapping.json index e9d180584a2e..e5a46005bdf5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/tag_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/tag_index_mapping.json @@ -179,7 +179,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json index 38ddf70819f3..551f8617c32f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/team_index_mapping.json @@ -132,7 +132,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_index_mapping.json index e09604289c2c..2963bfc6198f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_index_mapping.json @@ -1,6 +1,8 @@ { "settings": { - "index": {}, + "index": { + "max_ngram_diff": 1 + }, "analysis": { "normalizer": { "lowercase_normalizer": { @@ -74,9 +76,6 @@ ] } } - }, - "index": { - "max_ngram_diff": 1 } }, "mappings": { @@ -303,7 +302,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_resolution_status_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_resolution_status_index_mapping.json index c338987dc985..fe00642faa37 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_resolution_status_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_case_resolution_status_index_mapping.json @@ -587,7 +587,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -757,7 +758,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json index d02cbd44b608..4474e4aae062 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/test_suite_index_mapping.json @@ -1,6 +1,8 @@ { "settings": { - "index": {}, + "index": { + "max_ngram_diff": 1 + }, "analysis": { "normalizer": { "lowercase_normalizer": { @@ -74,9 +76,6 @@ ] } } - }, - "index": { - "max_ngram_diff": 1 } }, "mappings": { @@ -258,7 +257,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json index ed7173993235..44778085e932 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/topic_index_mapping.json @@ -622,7 +622,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/user_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/user_index_mapping.json index 855c6ce25436..b1bc133e8ce5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/user_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/user_index_mapping.json @@ -124,7 +124,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json index e208d6aa8e5d..d0999bfeadd3 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/jp/worksheet_index_mapping.json @@ -266,7 +266,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -541,7 +542,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json index 3a6b4292272e..fe40817dbfc4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_collection_index_mapping.json @@ -187,7 +187,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -269,7 +270,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json index 286fa335fb74..03708e6e6952 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_endpoint_index_mapping.json @@ -303,7 +303,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -563,7 +564,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_service_index_mapping.json index 3be0ac43516c..676f850aae71 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/api_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/api_service_index_mapping.json @@ -219,7 +219,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -362,7 +363,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json index c753bae4eabd..87fbbc97f2b1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/chart_index_mapping.json @@ -362,7 +362,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -554,7 +555,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json index 99b8091a736b..48ae62f8524c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/container_index_mapping.json @@ -719,7 +719,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json index f20daa1fcab5..90c8aa1ac9b5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_data_model_index_mapping.json @@ -196,7 +196,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -298,7 +299,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json index a98e727084e8..baa7f13629f5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_index_mapping.json @@ -471,7 +471,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -649,7 +650,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_service_index_mapping.json index eb32ecf5f194..dc1d7dc458c6 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/dashboard_service_index_mapping.json @@ -208,7 +208,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -361,7 +362,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json index ddf3383c1b78..a66771987905 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/data_products_index_mapping.json @@ -321,7 +321,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json index 28ddf6277ff6..ced4c8d14198 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_index_mapping.json @@ -187,7 +187,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -269,7 +270,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json index 1c6dc4fd99dd..9f93082a5e3e 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_schema_index_mapping.json @@ -196,7 +196,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -499,7 +500,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_service_index_mapping.json index 5b8b45a1de75..7f560a0ab2b2 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/database_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/database_service_index_mapping.json @@ -219,7 +219,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -365,7 +366,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json index be12a419f4e1..562622b02572 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/directory_index_mapping.json @@ -160,7 +160,8 @@ "analyzer": "om_analyzer" }, "serviceType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "service": { "properties": { @@ -175,7 +176,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -197,7 +199,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -224,7 +227,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -237,7 +241,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -307,7 +312,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -320,7 +326,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -360,7 +367,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -384,7 +392,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -403,7 +412,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json index 880557ca73ac..c7947056d53c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/domain_index_mapping.json @@ -302,7 +302,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/drive_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/drive_service_index_mapping.json index 64b9603d1cd9..725b7eefd004 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/drive_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/drive_service_index_mapping.json @@ -280,7 +280,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json index 7ba08002a671..1d431a82f022 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/file_index_mapping.json @@ -160,7 +160,8 @@ "analyzer": "om_analyzer" }, "serviceType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "service": { "properties": { @@ -175,7 +176,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -197,7 +199,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -224,7 +227,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -237,7 +241,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -362,7 +367,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -375,7 +381,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -415,7 +422,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -439,7 +447,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -458,7 +467,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_index_mapping.json index 08931778c2ae..b6a1d709adb5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_index_mapping.json @@ -327,7 +327,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_term_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_term_index_mapping.json index 6779763ead55..7e6338a083fb 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_term_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/glossary_term_index_mapping.json @@ -465,7 +465,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/ingestion_pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/ingestion_pipeline_index_mapping.json index 38a8601421fe..beb87c5daede 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/ingestion_pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/ingestion_pipeline_index_mapping.json @@ -335,7 +335,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/messaging_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/messaging_service_index_mapping.json index 4d4360bbd43f..e45b8d30c9f7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/messaging_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/messaging_service_index_mapping.json @@ -218,7 +218,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -361,7 +362,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/metadata_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/metadata_service_index_mapping.json index 39d7417e8fe7..5f247a5063f9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/metadata_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/metadata_service_index_mapping.json @@ -263,7 +263,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json index d255839eb1e8..aa31c4db633a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/metric_index_mapping.json @@ -397,7 +397,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json index 1e0a0a13f43e..ffcef7380738 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_index_mapping.json @@ -313,7 +313,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -639,7 +640,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_service_index_mapping.json index bc3d4d53811e..b9ea4967dbaa 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/mlmodel_service_index_mapping.json @@ -229,7 +229,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -362,7 +363,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json index 3fbe642d2e72..b8e4506bf020 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_index_mapping.json @@ -196,7 +196,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -542,7 +543,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_service_index_mapping.json index c7a604d4575c..43fb3c6bc553 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/pipeline_service_index_mapping.json @@ -190,7 +190,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -360,7 +361,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json index a066795f9eb8..9075df50a831 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_entity_index_mapping.json @@ -173,7 +173,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -606,7 +607,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_service_index_mapping.json index 3be0ac43516c..676f850aae71 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/search_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/search_service_index_mapping.json @@ -219,7 +219,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -362,7 +363,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/security_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/security_service_index_mapping.json index a06988289c3f..4998bf197159 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/security_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/security_service_index_mapping.json @@ -293,7 +293,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json index 307909569845..5126fbdf8774 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/spreadsheet_index_mapping.json @@ -160,7 +160,8 @@ "analyzer": "om_analyzer" }, "serviceType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "service": { "properties": { @@ -175,7 +176,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -197,7 +199,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -224,7 +227,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -237,7 +241,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -361,7 +366,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -374,7 +380,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -414,7 +421,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -438,7 +446,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -457,7 +466,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/storage_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/storage_service_index_mapping.json index 51fcea9302a0..4a773e192760 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/storage_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/storage_service_index_mapping.json @@ -214,7 +214,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -357,7 +358,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json index 37b25ea16527..8ff34d9325df 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/stored_procedure_index_mapping.json @@ -203,7 +203,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -492,7 +493,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json index 250dd4203077..628ea29ea095 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/table_index_mapping.json @@ -594,7 +594,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -712,7 +713,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_index_mapping.json index 477674113838..220d520dbe97 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_index_mapping.json @@ -596,7 +596,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_resolution_status_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_resolution_status_index_mapping.json index 38eadd24e585..b1ac9fec5a5a 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_resolution_status_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_resolution_status_index_mapping.json @@ -807,7 +807,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json index e667feb63890..95d6ff2e7982 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_case_result_index_mapping.json @@ -337,7 +337,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_suite_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_suite_index_mapping.json index f29f71806cca..bbc2728875a1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/test_suite_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/test_suite_index_mapping.json @@ -290,7 +290,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json index fbd5ae39dd6f..88606f63358d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/topic_index_mapping.json @@ -259,7 +259,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -475,7 +476,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json index 4784ac258309..25e61b5a4d23 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/ru/worksheet_index_mapping.json @@ -160,7 +160,8 @@ "analyzer": "om_analyzer" }, "serviceType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "service": { "properties": { @@ -175,7 +176,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -197,7 +199,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -224,7 +227,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -237,7 +241,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -313,7 +318,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -368,7 +374,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "fullyQualifiedName": { "type": "text" @@ -381,7 +388,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -421,7 +429,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -445,7 +454,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -464,7 +474,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json index e86da4b52861..8ef7bc9abae4 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_collection_index_mapping.json @@ -164,7 +164,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json index 068f66788a37..5379fac29e25 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_endpoint_index_mapping.json @@ -540,7 +540,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_service_index_mapping.json index d78a79a5e9d8..03f4bd5e7151 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/api_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/api_service_index_mapping.json @@ -186,7 +186,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json index c250ee0c43d2..4afd9470a412 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/chart_index_mapping.json @@ -300,7 +300,8 @@ "type": "keyword", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -372,7 +373,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -564,7 +566,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json index ed0dd7bbf526..1a6143e92df1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/container_index_mapping.json @@ -682,7 +682,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json index a74628fddf96..c0f61aad2e07 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_data_model_index_mapping.json @@ -381,7 +381,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -497,7 +498,8 @@ "normalizer": "lowercase_normalizer" }, "dataModelType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "columns": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json index 2a0bf0b4ca0a..f1ef5665ca11 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_index_mapping.json @@ -573,7 +573,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json index b8839a092d3d..38e5d72c69af 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/dashboard_service_index_mapping.json @@ -182,7 +182,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json index 17f0dc309d3c..f52d9bb5ce3d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/data_products_index_mapping.json @@ -486,7 +486,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json index 82b79b92a0cf..da40e43df134 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_index_mapping.json @@ -165,7 +165,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json index 9b5748f061ee..f038e6e8c5ec 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_schema_index_mapping.json @@ -174,7 +174,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -477,7 +478,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_service_index_mapping.json index 58cd48fb55c8..c34ad186dd41 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/database_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/database_service_index_mapping.json @@ -196,7 +196,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json index 79048659730a..5486da5b4ec0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/directory_index_mapping.json @@ -165,7 +165,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -440,7 +441,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json index 713e3668f894..934264b9bd30 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/domain_index_mapping.json @@ -234,7 +234,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/drive_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/drive_service_index_mapping.json index be7c335c69be..d57607c70327 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/drive_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/drive_service_index_mapping.json @@ -168,7 +168,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json index b59448d76373..9ecd80995724 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/file_index_mapping.json @@ -180,7 +180,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -455,7 +456,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_index_mapping.json index 4b16d41c17e3..cce5bca11511 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_index_mapping.json @@ -273,7 +273,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_term_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_term_index_mapping.json index dac786d82ed7..ad11ed7b76bf 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_term_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/glossary_term_index_mapping.json @@ -275,7 +275,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, @@ -443,7 +444,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json index d93e951a2aa6..924f719f885d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/ingestion_pipeline_index_mapping.json @@ -312,7 +312,8 @@ "tier": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -365,7 +366,8 @@ } }, "pipelineType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "pipelineStatuses": { "properties": { diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json index cf49c3bc5287..ea468032a33f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/messaging_service_index_mapping.json @@ -194,7 +194,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/metadata_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/metadata_service_index_mapping.json index 1efd07403428..9a6c70ce5138 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/metadata_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/metadata_service_index_mapping.json @@ -193,7 +193,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json index 12dc8aaf3fd5..a5f70c9c1fe3 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/metric_index_mapping.json @@ -427,7 +427,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json index 11c8edb1b19c..d26ef24b9b4b 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_index_mapping.json @@ -612,7 +612,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json index 95a597ba8a61..9d7fdf3e9fc9 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/mlmodel_service_index_mapping.json @@ -206,7 +206,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json index 23c504b3b145..6302fa52f90d 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_index_mapping.json @@ -507,7 +507,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json index 7d7ee6afe986..0a73b34de64c 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/pipeline_service_index_mapping.json @@ -165,7 +165,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json index 42ff631379d8..ce53111145be 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_entity_index_mapping.json @@ -121,7 +121,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -153,7 +154,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_service_index_mapping.json index d78a79a5e9d8..03f4bd5e7151 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/search_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/search_service_index_mapping.json @@ -186,7 +186,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/security_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/security_service_index_mapping.json index 4815af195e26..71e77aa4ef65 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/security_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/security_service_index_mapping.json @@ -89,7 +89,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -348,7 +349,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -399,7 +401,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", @@ -425,7 +428,8 @@ "tags": { "properties": { "tagFQN": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -452,7 +456,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" @@ -604,7 +609,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json index c2a609f00a7c..be70da2fdea1 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/spreadsheet_index_mapping.json @@ -165,7 +165,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -440,7 +441,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json index 34a246a52b55..ce978840e26f 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/storage_service_index_mapping.json @@ -189,7 +189,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json index e3d692961c18..de9916573e14 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/stored_procedure_index_mapping.json @@ -559,7 +559,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json index e547e962cfd4..e13ea9e69dd0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/table_index_mapping.json @@ -325,7 +325,8 @@ } }, "tableType": { - "type": "keyword" + "type": "keyword", + "normalizer": "lowercase_normalizer" }, "columns": { "properties": { @@ -853,7 +854,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/tag_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/tag_index_mapping.json index cbb68e2d20a7..add701d562a5 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/tag_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/tag_index_mapping.json @@ -185,7 +185,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" } } }, diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json index a2acb2cf10ac..3ba074810dfd 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/team_index_mapping.json @@ -122,7 +122,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_index_mapping.json index f178fadba0b3..40d7afde1fce 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_index_mapping.json @@ -1,6 +1,8 @@ { "settings": { - "index": {}, + "index": { + "max_ngram_diff": 1 + }, "analysis": { "normalizer": { "lowercase_normalizer": { @@ -64,9 +66,6 @@ ] } } - }, - "index": { - "max_ngram_diff": 1 } }, "mappings": { @@ -212,7 +211,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_resolution_status_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_resolution_status_index_mapping.json index b1d220eb110a..319969b15b01 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_resolution_status_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_resolution_status_index_mapping.json @@ -746,7 +746,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json index 905273b2debd..1946c80821dd 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_case_result_index_mapping.json @@ -327,7 +327,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json index 0d5e19e01b8f..8af00984edb7 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/test_suite_index_mapping.json @@ -226,7 +226,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json index 7b3001921d28..e72516690a91 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/topic_index_mapping.json @@ -458,7 +458,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/user_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/user_index_mapping.json index 7f8c8c01e67f..bc2f15451119 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/user_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/user_index_mapping.json @@ -92,7 +92,8 @@ "fields": { "keyword": { "type": "keyword", - "ignore_above": 256 + "ignore_above": 256, + "normalizer": "lowercase_normalizer" }, "ngram": { "type": "text", diff --git a/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json b/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json index f00d4e46fc19..9b1441ba82c0 100644 --- a/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json +++ b/openmetadata-spec/src/main/resources/elasticsearch/zh/worksheet_index_mapping.json @@ -241,7 +241,8 @@ "normalizer": "lowercase_normalizer", "ignore_above": 256 } - } + }, + "normalizer": "lowercase_normalizer" }, "displayName": { "type": "keyword", @@ -516,7 +517,8 @@ "type": "text", "analyzer": "om_analyzer" } - } + }, + "normalizer": "lowercase_normalizer" }, "labelType": { "type": "keyword" From 905a1f21efc5933b2439da0954499846fbffeb88 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 14:55:36 +0200 Subject: [PATCH 07/19] test(search): install analysis-ik and validate zh with real analyzers Add the third-party IK plugin (analysis-ik, ik_max_word/ik_smart) to the IT OpenSearch image alongside analysis-kuromoji, with the release URL derived from the base image tag so it always matches the OpenSearch version under test. Chinese mappings now index with their real analyzers in CI. Drop the analysis stripping from SearchConsumerFieldBehaviorIT entirely: all four languages now create and query with their real analyzers (en/ru built-in, jp kuromoji, zh IK), and the test asserts both plugins are installed. Every per-language search-consumer check now runs against real analyzers in one IT, with no need to run the whole integration suite once per language. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../it/server/SearchTestImages.java | 24 +++--- .../tests/SearchConsumerFieldBehaviorIT.java | 79 +++---------------- 2 files changed, 27 insertions(+), 76 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java index 36ebe1eba635..4ebc3544816c 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/server/SearchTestImages.java @@ -5,22 +5,25 @@ /** * Builds the OpenSearch test image with the language-analysis plugins baked in, so the integration - * tests can exercise non-English mapping languages (e.g. {@code analysis-kuromoji} for Japanese). + * tests can exercise every non-English mapping language: {@code analysis-kuromoji} for Japanese and + * {@code analysis-ik} ({@code ik_max_word}/{@code ik_smart}) for Chinese. * *

The plugins are installed unconditionally: a run that only uses the English mappings never - * references them, while a run configured for {@code jp} can create indexes whose text fields use - * {@code kuromoji_tokenizer}. This is what lets the search IT suite catch per-language + * references them, while a run configured for {@code jp}/{@code zh} can create indexes whose text + * fields use those analyzers. This is what lets the search IT suite catch per-language * mapping/analyzer drift — the jp mappings referencing undefined analyzers went unnoticed precisely * because CI only ever ran English on a vanilla image. * - *

Chinese ({@code zh}) uses the third-party IK plugin ({@code analysis-ik}); it is intentionally - * not installed here because it ships only as a version-matched release URL that would break the - * image build if it became unavailable. Add it separately when zh coverage is needed. + *

{@code analysis-ik} is third-party and ships only as a version-matched release URL; the URL is + * derived from the base image tag so it always matches the OpenSearch version being tested. */ public final class SearchTestImages { private static final String OPENSEARCH_BASE_REFERENCE = "opensearchproject/opensearch"; - private static final String ANALYSIS_PLUGINS = "analysis-kuromoji"; + private static final String PLUGIN_INSTALL = + "/usr/share/opensearch/bin/opensearch-plugin install --batch "; + private static final String IK_PLUGIN_URL_TEMPLATE = + "https://release.infinilabs.com/analysis-ik/stable/opensearch-analysis-ik-%s.zip"; private SearchTestImages() {} @@ -29,15 +32,16 @@ private SearchTestImages() {} * image is built once per run and reused via Docker's layer cache. */ public static DockerImageName openSearchWithAnalysisPlugins(String baseImage) { + String version = baseImage.substring(baseImage.lastIndexOf(':') + 1); + String ikPluginUrl = String.format(IK_PLUGIN_URL_TEMPLATE, version); String builtImage = new ImageFromDockerfile() .withDockerfileFromBuilder( builder -> builder .from(baseImage) - .run( - "/usr/share/opensearch/bin/opensearch-plugin install --batch " - + ANALYSIS_PLUGINS) + .run(PLUGIN_INSTALL + "analysis-kuromoji") + .run(PLUGIN_INSTALL + ikPluginUrl) .build()) .get(); return DockerImageName.parse(builtImage).asCompatibleSubstituteFor(OPENSEARCH_BASE_REFERENCE); diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java index c1c3c7e2e22d..58133a174434 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java @@ -5,7 +5,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; import java.io.IOException; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -37,14 +36,12 @@ * reported as a broken feature in a specific language (e.g. "Domain filter is broken in * [jp]"), which is what an operator can act on — not "index is missing a mapping". * - *

This is the search-side coverage the suite lacked for non-English languages — without the need - * to run the whole integration-test suite once per language. The container is built with the - * language-analysis plugins ({@link SearchTestImages}), so en, ru and jp index with their - * real analyzers (jp uses {@code kuromoji_tokenizer}); creating the real jp mapping here - * therefore also catches analyzer drift (fields referencing analyzers the mapping never defined). - * zh uses the third-party {@code analysis-ik} plugin, which is not installed, so zh alone has its - * analysis stripped and is validated structurally — the consumer fields are {@code keyword}/{@code - * nested} and analyzer-independent, so every assertion still holds. + *

This is the search-side coverage the suite lacked for non-English languages — without running + * the whole integration-test suite once per language. The container is built with the + * language-analysis plugins ({@link SearchTestImages}), so every language indexes with its + * real analyzers (jp uses {@code kuromoji_tokenizer} via analysis-kuromoji, zh uses {@code + * ik_max_word}/{@code ik_smart} via analysis-ik). Creating the real mappings here therefore also + * catches analyzer drift — e.g. fields referencing analyzers the mapping never defined. * *

It would have caught the {@code jp/topic} mapping dropping top-level {@code domains} (the * domain-filter assertion returns zero hits for {@code jp}) and the jp analyzer references that made @@ -97,16 +94,10 @@ void setUp() throws Exception { mapper = new ObjectMapper(); for (String language : LANGUAGES) { - boolean stripAnalysis = needsAnalysisStrip(language); - createIndex( - topicIndex(language), - "/elasticsearch/" + language + "/topic_index_mapping.json", - stripAnalysis); + createIndex(topicIndex(language), "/elasticsearch/" + language + "/topic_index_mapping.json"); indexDocument(topicIndex(language), TOPIC_ID, topicDocument()); createIndex( - testCaseIndex(language), - "/elasticsearch/" + language + "/test_case_index_mapping.json", - stripAnalysis); + testCaseIndex(language), "/elasticsearch/" + language + "/test_case_index_mapping.json"); indexDocument(testCaseIndex(language), TEST_CASE_ID, testCaseDocument()); } } @@ -181,16 +172,16 @@ void dataQualityStatusAggregationHasBucketInAllLanguages() throws Exception { } @Test - void japaneseAnalysisPluginIsInstalled() throws Exception { + void analysisPluginsInstalled() throws Exception { try (var response = openSearchClient .generic() .execute(Requests.builder().method("GET").endpoint("/_cat/plugins").build())) { String plugins = response.getBody().map(b -> b.bodyAsString()).orElse(""); assertTrue( - plugins.contains("analysis-kuromoji"), - "The integration-test OpenSearch image must ship analysis-kuromoji so jp mappings index " - + "with their real kuromoji analyzer instead of a stripped fallback. Installed: " + plugins.contains("analysis-kuromoji") && plugins.contains("analysis-ik"), + "The integration-test OpenSearch image must ship analysis-kuromoji (jp) and analysis-ik " + + "(zh) so jp/zh mappings index with their real analyzers. Installed: " + plugins); } } @@ -263,17 +254,13 @@ private JsonNode runSearch(String index, String body) throws Exception { } } - private void createIndex(String index, String mappingResource, boolean stripAnalysis) - throws Exception { + private void createIndex(String index, String mappingResource) throws Exception { String rawMapping; try (InputStream in = getClass().getResourceAsStream(mappingResource)) { assertNotNull(in, "Mapping resource not found on classpath: " + mappingResource); rawMapping = new String(in.readAllBytes(), StandardCharsets.UTF_8); } String enriched = OsUtils.enrichIndexMappingForOpenSearch(rawMapping); - if (stripAnalysis) { - enriched = stripLanguageAnalysis(enriched); - } try (var response = openSearchClient .generic() @@ -387,44 +374,4 @@ private String topicIndex(String language) { private String testCaseIndex(String language) { return "behavior_testcase_" + language; } - - /** - * Languages whose analyzers are available in the IT image keep their real analysis (en/ru are - * built-in, jp uses the installed {@code analysis-kuromoji}). Only zh — which needs the - * uninstalled third-party {@code analysis-ik} — has its analysis stripped, so its index is still - * creatable for structural validation. The consumer fields asserted here are {@code keyword}/{@code - * nested} and analyzer-independent, so stripping zh does not weaken any assertion. - */ - private boolean needsAnalysisStrip(String language) { - return "zh".equals(language); - } - - /** - * Remove every analysis setting and field-level {@code analyzer}/{@code search_analyzer}/{@code - * normalizer} reference so an index is creatable without its language-analysis plugin (used only - * for zh). What remains is the real field structure, so structural drift — e.g. a language mapping - * dropping a top-level field — is still caught. - */ - private String stripLanguageAnalysis(String mappingJson) throws Exception { - JsonNode root = mapper.readTree(mappingJson); - if (root.path("settings") instanceof ObjectNode settings) { - settings.remove("analysis"); - if (settings.path("index") instanceof ObjectNode indexSettings) { - indexSettings.remove("analysis"); - } - } - stripAnalysisReferences(root.path("mappings")); - return mapper.writeValueAsString(root); - } - - private void stripAnalysisReferences(JsonNode node) { - if (node instanceof ObjectNode object) { - object.remove("analyzer"); - object.remove("search_analyzer"); - object.remove("normalizer"); - object.forEach(this::stripAnalysisReferences); - } else if (node.isArray()) { - node.forEach(this::stripAnalysisReferences); - } - } } From f4f236e9ffaadb1ef3f064fe2bb5ffdc42c4a5c0 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 15:36:05 +0200 Subject: [PATCH 08/19] test(search): derive validated languages from the IndexMappingLanguage enum Replace the hardcoded ["en","jp","ru","zh"] list with IndexMappingLanguage.values() mapped through the same toString().toLowerCase() the loader uses to pick a per-language mapping file. A newly supported language is then exercised automatically, and a declared language with no mapping files fails fast in createIndex. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../it/tests/SearchConsumerFieldBehaviorIT.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java index 58133a174434..4d17f606222f 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java @@ -10,7 +10,9 @@ import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; +import java.util.Locale; import java.util.Map; import org.apache.hc.core5.http.HttpHost; import org.junit.jupiter.api.AfterAll; @@ -18,6 +20,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.TestInstance; import org.openmetadata.it.server.SearchTestImages; +import org.openmetadata.schema.type.IndexMappingLanguage; import org.openmetadata.service.search.opensearch.OsUtils; import org.opensearch.testcontainers.OpensearchContainer; import org.testcontainers.junit.jupiter.Container; @@ -54,7 +57,13 @@ @TestInstance(TestInstance.Lifecycle.PER_CLASS) class SearchConsumerFieldBehaviorIT { - private static final List LANGUAGES = List.of("en", "jp", "ru", "zh"); + // Discovered from the schema-defined language registry (IndexMappingLanguage) that the loader + // itself uses to pick a per-language mapping file, so a newly added language is exercised + // automatically — and a declared language with no mapping files fails fast in createIndex. + private static final List LANGUAGES = + Arrays.stream(IndexMappingLanguage.values()) + .map(language -> language.toString().toLowerCase(Locale.ROOT)) + .toList(); private static final String TAG_FQN = "PII.Sensitive"; private static final String TIER_FQN = "Tier.Tier1"; From 350b41bf60e902bddaccf9f17e17e02cf1fd32f2 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 15:57:51 +0200 Subject: [PATCH 09/19] test(search): add Data Quality and Incident Manager per-language coverage Extend SearchConsumerFieldBehaviorIT with the real queries the DQ dashboard, test case list and Incident Manager run, validated across every language with real analyzers: - test_case index: nested testSuites.id filter, entityLink.nonNormalized per-entity execution-summary aggregation, dataQualityDimension and testPlatforms filters. - test_case_resolution_status index (new): testCaseResolutionStatusType, testCaseResolutionStatusDetails.assignee.name, and the testCase.fullyQualifiedName.keyword / testCase.entityFQN.keyword filters. Field paths and query shapes mirror the backend SearchListFilter / TestSuiteRepository and the UI DQ/incident filter builders. 16 capabilities now run per language. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/SearchConsumerFieldBehaviorIT.java | 166 ++++++++++++++++-- 1 file changed, 153 insertions(+), 13 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java index 4d17f606222f..1f874b6c714f 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java @@ -52,6 +52,12 @@ * (RBAC isOwner), term filters for tags/tier/certification/domains (Explore facets, RBAC, Data * Quality), a terms aggregation for {@code testCaseResult.testCaseStatus} (the Data Quality * execution summary), and a keyword term on {@code fqnParts} (hierarchical search / autocomplete). + * + *

Coverage spans the Explore/RBAC fields on a data-asset index (topic) plus the Data Quality + * test-case index — status aggregation, the nested {@code testSuites.id} filter, the {@code + * entityLink.nonNormalized} per-entity summary aggregation, and the dimension/platform filters — and + * the Incident Manager resolution-status index — status type, assignee, and the {@code + * testCase.fullyQualifiedName.keyword}/{@code testCase.entityFQN.keyword} filters. */ @Testcontainers @TestInstance(TestInstance.Lifecycle.PER_CLASS) @@ -77,6 +83,16 @@ class SearchConsumerFieldBehaviorIT { private static final String TEST_CASE_ID = "44444444-4444-4444-4444-444444444444"; private static final String TEST_CASE_FQN = "svc.ns.orders.columns.amount.notNull"; private static final String TEST_CASE_STATUS = "Success"; + private static final String ENTITY_FQN = "svc.ns.orders"; + private static final String ENTITY_LINK = "<#E::table::svc.ns.orders>"; + private static final String DQ_DIMENSION = "Completeness"; + private static final String TEST_PLATFORM = "DBT"; + private static final String TEST_SUITE_ID = "55555555-5555-5555-5555-555555555555"; + private static final String RESOLUTION_ID = "66666666-6666-6666-6666-666666666666"; + private static final String RESOLUTION_STATUS_TYPE = "Assigned"; + private static final String ASSIGNEE_ID = "77777777-7777-7777-7777-777777777777"; + private static final String ASSIGNEE_NAME = "john"; + private static final String STATE_ID = "88888888-8888-8888-8888-888888888888"; @Container static OpensearchContainer opensearch = @@ -108,6 +124,10 @@ void setUp() throws Exception { createIndex( testCaseIndex(language), "/elasticsearch/" + language + "/test_case_index_mapping.json"); indexDocument(testCaseIndex(language), TEST_CASE_ID, testCaseDocument()); + createIndex( + resolutionIndex(language), + "/elasticsearch/" + language + "/test_case_resolution_status_index_mapping.json"); + indexDocument(resolutionIndex(language), RESOLUTION_ID, resolutionStatusDocument()); } } @@ -180,6 +200,86 @@ void dataQualityStatusAggregationHasBucketInAllLanguages() throws Exception { TEST_CASE_STATUS)); } + @Test + void testSuiteNestedFilterReturnsTestCaseInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Test suite filter (Data Quality test list + execution summary; nested testSuites.id)", + language -> + hits( + testCaseIndex(language), + nestedTermQuery("testSuites", "testSuites.id", TEST_SUITE_ID)) + == 1); + } + + @Test + void entityLinkSummaryAggregationHasBucketInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Data Quality per-entity execution summary (entityLink.nonNormalized aggregation)", + language -> + aggregationHasBucket( + testCaseIndex(language), + termsAggregation("links", "entityLink.nonNormalized"), + "links", + ENTITY_LINK)); + } + + @Test + void dataQualityDimensionFilterReturnsTestCaseInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Data Quality dimension filter (test list + DQ report by dimension)", + language -> + hits(testCaseIndex(language), termQuery("dataQualityDimension", DQ_DIMENSION)) == 1); + } + + @Test + void testPlatformFilterReturnsTestCaseInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Test platform filter (Data Quality test list)", + language -> hits(testCaseIndex(language), termQuery("testPlatforms", TEST_PLATFORM)) == 1); + } + + @Test + void incidentStatusFilterReturnsIncidentInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Incident Manager status filter (testCaseResolutionStatusType)", + language -> + hits( + resolutionIndex(language), + termQuery("testCaseResolutionStatusType", RESOLUTION_STATUS_TYPE)) + == 1); + } + + @Test + void incidentAssigneeFilterReturnsIncidentInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Incident Manager assignee filter (testCaseResolutionStatusDetails.assignee.name)", + language -> + hits( + resolutionIndex(language), + termQuery("testCaseResolutionStatusDetails.assignee.name", ASSIGNEE_NAME)) + == 1); + } + + @Test + void incidentTestCaseFqnFilterReturnsIncidentInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Incident Manager test-case filter (testCase.fullyQualifiedName.keyword)", + language -> + hits( + resolutionIndex(language), + termQuery("testCase.fullyQualifiedName.keyword", TEST_CASE_FQN)) + == 1); + } + + @Test + void incidentOriginEntityFilterReturnsIncidentInAllLanguages() throws Exception { + assertFeatureWorksInAllLanguages( + "Incident Manager origin-entity filter (testCase.entityFQN.keyword)", + language -> + hits(resolutionIndex(language), termQuery("testCase.entityFQN.keyword", ENTITY_FQN)) + == 1); + } + @Test void analysisPluginsInstalled() throws Exception { try (var response = @@ -336,19 +436,55 @@ private String topicDocument() throws Exception { private String testCaseDocument() throws Exception { Map document = - Map.of( - "id", - TEST_CASE_ID, - "name", - "amount_not_null", - "fullyQualifiedName", - TEST_CASE_FQN, - "deleted", - false, - "entityType", - "testCase", - "testCaseResult", - Map.of("testCaseStatus", TEST_CASE_STATUS, "timestamp", 1700000000000L)); + Map.ofEntries( + Map.entry("id", TEST_CASE_ID), + Map.entry("name", "amount_not_null"), + Map.entry("fullyQualifiedName", TEST_CASE_FQN), + Map.entry("deleted", false), + Map.entry("entityType", "testCase"), + Map.entry("entityFQN", ENTITY_FQN), + Map.entry("originEntityFQN", ENTITY_FQN), + Map.entry("entityLink", ENTITY_LINK), + Map.entry("dataQualityDimension", DQ_DIMENSION), + Map.entry("testPlatforms", List.of(TEST_PLATFORM)), + Map.entry( + "testSuites", + List.of( + Map.of( + "id", + TEST_SUITE_ID, + "name", + "suite", + "fullyQualifiedName", + TEST_CASE_FQN + ".suite"))), + Map.entry( + "testCaseResult", + Map.of("testCaseStatus", TEST_CASE_STATUS, "timestamp", 1700000000000L))); + return mapper.writeValueAsString(document); + } + + private String resolutionStatusDocument() throws Exception { + Map document = + Map.ofEntries( + Map.entry("id", RESOLUTION_ID), + Map.entry("entityType", "testCaseResolutionStatus"), + Map.entry("testCaseResolutionStatusType", RESOLUTION_STATUS_TYPE), + Map.entry( + "testCaseResolutionStatusDetails", + Map.of( + "assignee", Map.of("id", ASSIGNEE_ID, "type", "user", "name", ASSIGNEE_NAME))), + Map.entry( + "testCase", + Map.of( + "name", + "amount_not_null", + "fullyQualifiedName", + TEST_CASE_FQN, + "entityFQN", + ENTITY_FQN)), + Map.entry("stateId", STATE_ID), + Map.entry("@timestamp", 1700000000000L), + Map.entry("timestamp", 1700000000000L)); return mapper.writeValueAsString(document); } @@ -383,4 +519,8 @@ private String topicIndex(String language) { private String testCaseIndex(String language) { return "behavior_testcase_" + language; } + + private String resolutionIndex(String language) { + return "behavior_resolution_" + language; + } } From a798fa967a53237ac41779c2a3cfec5750b5538d Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 16:26:18 +0200 Subject: [PATCH 10/19] feat(search): add computeDrift to IndexMappingVersionTracker Adds MappingDriftState enum (CURRENT/STALE/UNTRACKED) and computeDrift() method to classify each entity's index mapping against the stored hash, enabling /system/validate to detect stale deployed indices. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../search/IndexMappingVersionTracker.java | 31 +++++++++++++ .../IndexMappingVersionTrackerTest.java | 44 +++++++++++++++++++ 2 files changed, 75 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java index a2971f802642..7c16d3615205 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java @@ -7,9 +7,11 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.search.IndexMapping; @@ -32,6 +34,12 @@ public IndexMappingVersionTracker(CollectionDAO daoCollection, String version, S this.updatedBy = updatedBy; } + public enum MappingDriftState { + CURRENT, + STALE, + UNTRACKED + } + public List getChangedMappings() throws IOException { List changedMappings = new ArrayList<>(); Map storedHashes = getStoredMappingHashes(); @@ -57,6 +65,29 @@ public List getChangedMappings() throws IOException { return changedMappings; } + public Map computeDrift() throws IOException { + Map storedHashes = getStoredMappingHashes(); + Map currentMappings = computeCurrentMappings(); + Map drift = new HashMap<>(); + for (Map.Entry entry : currentMappings.entrySet()) { + String storedHash = storedHashes.get(entry.getKey()); + drift.put(entry.getKey(), classifyState(storedHash, entry.getValue().hash())); + } + return Collections.unmodifiableMap(drift); + } + + private MappingDriftState classifyState(String storedHash, String currentHash) { + MappingDriftState state; + if (storedHash == null) { + state = MappingDriftState.UNTRACKED; + } else if (Objects.equals(storedHash, currentHash)) { + state = MappingDriftState.CURRENT; + } else { + state = MappingDriftState.STALE; + } + return state; + } + public void updateMappingVersions() throws IOException { Map currentMappings = computeCurrentMappings(); long updatedAt = System.currentTimeMillis(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java index d936e8f792a4..ba04352bcaa0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java @@ -416,6 +416,50 @@ void hashesAreStableAcrossMultipleComputations() throws IOException { } } + @Test + void computeDriftClassifiesCurrentStaleAndUntracked() throws IOException { + Map mappings = + buildMappingsFromPairs( + "table", "/elasticsearch/%s/table_index_mapping.json", + "glossaryTerm", "/elasticsearch/%s/glossary_term_index_mapping.json", + "domain", "/elasticsearch/%s/domain_index_mapping.json"); + try (var loaderMock = mockStatic(IndexMappingLoader.class)) { + loaderMock.when(IndexMappingLoader::getInstance).thenReturn(indexMappingLoader); + when(indexMappingLoader.getIndexMapping()).thenReturn(mappings); + + IndexMappingVersionTracker tracker = + new IndexMappingVersionTracker(collectionDAO, "1.2.3", "tester"); + + // Stamp all, capture table's real hash to mark it CURRENT. + tracker.updateMappingVersions(); + verify(indexMappingVersionDAO, times(3)) + .upsertIndexMappingVersion( + entityTypeCaptor.capture(), + hashCaptor.capture(), + anyString(), + anyString(), + anyLong(), + anyString()); + Map realHashes = new HashMap<>(); + for (int i = 0; i < entityTypeCaptor.getAllValues().size(); i++) { + realHashes.put(entityTypeCaptor.getAllValues().get(i), hashCaptor.getAllValues().get(i)); + } + + // table = current hash, glossaryTerm = stale hash, domain = not stored (untracked). + when(indexMappingVersionDAO.getAllMappingVersions()) + .thenReturn( + List.of( + new IndexMappingVersionDAO.IndexMappingVersion("table", realHashes.get("table")), + new IndexMappingVersionDAO.IndexMappingVersion("glossaryTerm", "stale-hash"))); + + Map drift = tracker.computeDrift(); + + assertEquals(IndexMappingVersionTracker.MappingDriftState.CURRENT, drift.get("table")); + assertEquals(IndexMappingVersionTracker.MappingDriftState.STALE, drift.get("glossaryTerm")); + assertEquals(IndexMappingVersionTracker.MappingDriftState.UNTRACKED, drift.get("domain")); + } + } + private static Set diff(Set expected, Set actual) { Set missing = new TreeSet<>(expected); missing.removeAll(actual); From fccf2ac6dee32c1c5a16057dbfbf364b7fa11a57 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 16:48:27 +0200 Subject: [PATCH 11/19] feat(search): add per-subset stamping to IndexMappingVersionTracker Introduce `updateMappingVersions(Collection)` overload and extract shared `stamp()` helper so only the explicitly provided entity types are persisted; the no-arg method continues to stamp all entities as before. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../search/IndexMappingVersionTracker.java | 25 +++++++++++++----- .../IndexMappingVersionTrackerTest.java | 26 +++++++++++++++++++ 2 files changed, 45 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java index 7c16d3615205..98a402fde324 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexMappingVersionTracker.java @@ -7,6 +7,7 @@ import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -89,22 +90,34 @@ private MappingDriftState classifyState(String storedHash, String currentHash) { } public void updateMappingVersions() throws IOException { + stamp(computeCurrentMappings()); + } + + public void updateMappingVersions(Collection entityTypes) throws IOException { Map currentMappings = computeCurrentMappings(); - long updatedAt = System.currentTimeMillis(); + Map subset = new HashMap<>(); + for (String entityType : entityTypes) { + MappingEntry mappingEntry = currentMappings.get(entityType); + if (mappingEntry != null) { + subset.put(entityType, mappingEntry); + } + } + stamp(subset); + } - for (Map.Entry entry : currentMappings.entrySet()) { - String entityType = entry.getKey(); + private void stamp(Map mappings) { + long updatedAt = System.currentTimeMillis(); + for (Map.Entry entry : mappings.entrySet()) { MappingEntry mappingEntry = entry.getValue(); - indexMappingVersionDAO.upsertIndexMappingVersion( - entityType, + entry.getKey(), mappingEntry.hash(), JsonUtils.pojoToJson(mappingEntry.json()), version, updatedAt, updatedBy); } - LOG.info("Updated index mapping versions for {} entities", currentMappings.size()); + LOG.info("Updated index mapping versions for {} entities", mappings.size()); } private Map getStoredMappingHashes() { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java index ba04352bcaa0..3f29dffd09c3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/IndexMappingVersionTrackerTest.java @@ -460,6 +460,32 @@ void computeDriftClassifiesCurrentStaleAndUntracked() throws IOException { } } + @Test + void updateMappingVersionsForSubsetStampsOnlyGivenEntities() throws IOException { + Map mappings = + buildMappingsFromPairs( + "table", "/elasticsearch/%s/table_index_mapping.json", + "glossaryTerm", "/elasticsearch/%s/glossary_term_index_mapping.json", + "domain", "/elasticsearch/%s/domain_index_mapping.json"); + try (var loaderMock = mockStatic(IndexMappingLoader.class)) { + loaderMock.when(IndexMappingLoader::getInstance).thenReturn(indexMappingLoader); + when(indexMappingLoader.getIndexMapping()).thenReturn(mappings); + + new IndexMappingVersionTracker(collectionDAO, "1.2.3", "tester") + .updateMappingVersions(List.of("table", "domain")); + + verify(indexMappingVersionDAO, times(2)) + .upsertIndexMappingVersion( + entityTypeCaptor.capture(), + anyString(), + anyString(), + eq("1.2.3"), + anyLong(), + eq("tester")); + assertEquals(Set.of("domain", "table"), new TreeSet<>(entityTypeCaptor.getAllValues())); + } + } + private static Set diff(Set expected, Set actual) { Set missing = new TreeSet<>(expected); missing.removeAll(actual); From cddcb553519ff9127bd5fae613794c0462c0e9d4 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 16:55:20 +0200 Subject: [PATCH 12/19] feat(search): add reindex-status classification to SystemRepository Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/SystemRepository.java | 42 +++++++++ .../SystemRepositoryReindexStatusTest.java | 87 +++++++++++++++++++ 2 files changed, 129 insertions(+) create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index dd25c6d76bca..0fdbff88ec7c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -18,6 +18,7 @@ import jakarta.json.JsonValue; import jakarta.ws.rs.core.Response; import java.util.ArrayList; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Map; @@ -79,6 +80,7 @@ import org.openmetadata.service.logstorage.LogStorageInterface; import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.IndexMappingVersionTracker.MappingDriftState; import org.openmetadata.service.search.SearchConsumerFields; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.vector.client.EmbeddingClient; @@ -847,6 +849,46 @@ private StepValidation getSearchValidation(OpenMetadataApplicationConfig applica } } + public record ReindexStatus(List stalePending, int untrackedCount) {} + + static ReindexStatus classifyReindexStatus( + Map drift, Set existingIndexes) { + List stalePending = new ArrayList<>(); + int untrackedCount = 0; + for (Map.Entry entry : drift.entrySet()) { + if (existingIndexes.contains(entry.getKey())) { + if (entry.getValue() == MappingDriftState.STALE) { + stalePending.add(entry.getKey()); + } else if (entry.getValue() == MappingDriftState.UNTRACKED) { + untrackedCount++; + } + } + } + Collections.sort(stalePending); + return new ReindexStatus(stalePending, untrackedCount); + } + + static String buildReindexStatusMessage(ReindexStatus status) { + String message; + if (status.stalePending().isEmpty()) { + message = "All deployed indexes were built from the current code mappings."; + if (status.untrackedCount() > 0) { + message += + String.format( + " %d index(es) are not yet version-tracked; run a reindex to enable drift " + + "detection.", + status.untrackedCount()); + } + } else { + message = + String.format( + "WARNING: %d deployed index(es) were built from an older code mapping and need a " + + "reindex: %s", + status.stalePending().size(), status.stalePending()); + } + return message; + } + @VisibleForTesting List findMissingIndexes(SearchRepository searchRepository) { List missing = new ArrayList<>(); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java new file mode 100644 index 000000000000..a07feaec15b8 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryReindexStatusTest.java @@ -0,0 +1,87 @@ +/* + * Copyright 2024 Collate. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.openmetadata.service.jdbi3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.util.List; +import java.util.Map; +import java.util.Set; +import org.junit.jupiter.api.Test; +import org.openmetadata.service.jdbi3.SystemRepository.ReindexStatus; +import org.openmetadata.service.search.IndexMappingVersionTracker.MappingDriftState; + +class SystemRepositoryReindexStatusTest { + + @Test + void staleEntityWithLiveIndexIsReportedPending() { + Map drift = Map.of("table", MappingDriftState.STALE); + ReindexStatus status = SystemRepository.classifyReindexStatus(drift, Set.of("table")); + assertEquals(1, status.stalePending().size()); + assertTrue(status.stalePending().contains("table")); + } + + @Test + void missingIndexIsSkippedSoItIsNotDoubleReported() { + Map drift = Map.of("table", MappingDriftState.STALE); + ReindexStatus status = SystemRepository.classifyReindexStatus(drift, Set.of()); + assertTrue(status.stalePending().isEmpty()); + assertEquals(0, status.untrackedCount()); + } + + @Test + void untrackedEntityWithLiveIndexIsNotedNotFailed() { + Map drift = Map.of("table", MappingDriftState.UNTRACKED); + ReindexStatus status = SystemRepository.classifyReindexStatus(drift, Set.of("table")); + assertTrue(status.stalePending().isEmpty()); + assertEquals(1, status.untrackedCount()); + } + + @Test + void currentEntityProducesNoFinding() { + Map drift = Map.of("table", MappingDriftState.CURRENT); + ReindexStatus status = SystemRepository.classifyReindexStatus(drift, Set.of("table")); + assertTrue(status.stalePending().isEmpty()); + assertEquals(0, status.untrackedCount()); + } + + @Test + void messageListsStalePendingEntities() { + ReindexStatus status = new ReindexStatus(List.of("dashboard", "table"), 0); + String message = SystemRepository.buildReindexStatusMessage(status); + assertTrue(message.contains("dashboard")); + assertTrue(message.contains("table")); + assertTrue(message.toLowerCase().contains("reindex")); + } + + @Test + void cleanMessageMentionsUntrackedNote() { + ReindexStatus status = new ReindexStatus(List.of(), 3); + String message = SystemRepository.buildReindexStatusMessage(status); + assertTrue(message.contains("3")); + assertTrue(message.toLowerCase().contains("version-tracked")); + } + + @Test + void stalePendingIsReturnedInSortedOrder() { + Map drift = + Map.of( + "topic", MappingDriftState.STALE, + "chart", MappingDriftState.STALE, + "dashboard", MappingDriftState.STALE); + ReindexStatus status = + SystemRepository.classifyReindexStatus(drift, Set.of("topic", "chart", "dashboard")); + assertEquals(List.of("chart", "dashboard", "topic"), status.stalePending()); + } +} From 63356f250ac1b29e0dd97e5f5631976092c5d4dc Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:05:59 +0200 Subject: [PATCH 13/19] test(search): validate per-language text analyzers with native-script tokens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a per-language _analyze check on the description field (kuromoji for jp, IK for zh, stemmer for en/ru) asserting native-script text segments into the token the analyzer must produce (東京 / 销售 / продаж / revenue). Keyword and identifier fields are language-neutral and never exercise the text analyzers, so this is the only assertion that the per-language analyzers — the whole reason the per-language mappings exist — actually work. Tokens were verified against the real field analyzers (e.g. ru stems продажи -> продаж; jp drops the の particle). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../tests/SearchConsumerFieldBehaviorIT.java | 56 +++++++++ ...ystemRepositoryMappingConsistencyTest.java | 119 ------------------ 2 files changed, 56 insertions(+), 119 deletions(-) delete mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java index 1f874b6c714f..7ab60fda9a11 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/SearchConsumerFieldBehaviorIT.java @@ -94,6 +94,20 @@ class SearchConsumerFieldBehaviorIT { private static final String ASSIGNEE_NAME = "john"; private static final String STATE_ID = "88888888-8888-8888-8888-888888888888"; + // Native-script sample text per language plus a token its analyzer must produce. + // Keyword/identifier + // fields are language-neutral and never exercise the text analyzers; this is the only check that + // the per-language analyzers actually work — the reason the per-language mappings exist at all. + // jp/zh require kuromoji/IK to segment CJK (the plain standard analyzer splits it into single + // characters, never the 2-char tokens 東京 / 销售); ru requires the Russian stemmer (продажи -> + // продаж); en is the latin baseline. + private static final Map LANGUAGE_SAMPLE_TEXT = + Map.of( + "en", new String[] {"Monthly revenue report", "revenue"}, + "ru", new String[] {"Ежемесячные продажи", "продаж"}, + "jp", new String[] {"東京タワーの売上", "東京"}, + "zh", new String[] {"北京大学的销售报表", "销售"}); + @Container static OpensearchContainer opensearch = new OpensearchContainer<>( @@ -295,6 +309,25 @@ void analysisPluginsInstalled() throws Exception { } } + @Test + void languageAnalyzerSegmentsNativeTextInEachLanguage() throws Exception { + List broken = new ArrayList<>(); + for (Map.Entry entry : LANGUAGE_SAMPLE_TEXT.entrySet()) { + String language = entry.getKey(); + String text = entry.getValue()[0]; + String expectedToken = entry.getValue()[1]; + List tokens = analyzeField(topicIndex(language), "description", text); + if (!tokens.contains(expectedToken)) { + broken.add(language + " expected token '" + expectedToken + "' but got " + tokens); + } + } + assertTrue( + broken.isEmpty(), + "The per-language text analyzer did not segment native-script text into the expected token " + + "— full-text search in that language is broken: " + + broken); + } + @FunctionalInterface private interface LanguageProbe { boolean passes(String language) throws Exception; @@ -348,6 +381,29 @@ private List bucketKeys(String index, String aggBody, String aggName) th return keys; } + private List analyzeField(String index, String field, String text) throws Exception { + String body = mapper.writeValueAsString(Map.of("field", field, "text", text)); + try (var response = + openSearchClient + .generic() + .execute( + Requests.builder() + .method("POST") + .endpoint("/" + index + "/_analyze") + .json(body) + .build())) { + JsonNode tokens = + mapper + .readTree(response.getBody().map(b -> b.bodyAsString()).orElse("{}")) + .path("tokens"); + List result = new ArrayList<>(); + for (JsonNode token : tokens) { + result.add(token.path("token").asText()); + } + return result; + } + } + private JsonNode runSearch(String index, String body) throws Exception { try (var response = openSearchClient diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java deleted file mode 100644 index f911befd22a4..000000000000 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SystemRepositoryMappingConsistencyTest.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2024 Collate. - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.openmetadata.service.jdbi3; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.junit.jupiter.api.Assertions.assertTrue; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.when; - -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.MockedStatic; -import org.openmetadata.search.IndexMapping; -import org.openmetadata.service.Entity; -import org.openmetadata.service.jdbi3.CollectionDAO.SystemDAO; -import org.openmetadata.service.migration.MigrationValidationClient; -import org.openmetadata.service.search.SearchConsumerFields; -import org.openmetadata.service.search.SearchRepository; - -class SystemRepositoryMappingConsistencyTest { - - private MockedStatic entityMock; - private MockedStatic migrationMock; - private SearchRepository searchRepository; - private SystemRepository systemRepository; - - @BeforeEach - void setup() { - entityMock = mockStatic(Entity.class); - migrationMock = mockStatic(MigrationValidationClient.class); - - CollectionDAO collectionDAO = mock(CollectionDAO.class); - SystemDAO systemDAO = mock(SystemDAO.class); - when(collectionDAO.systemDAO()).thenReturn(systemDAO); - entityMock.when(Entity::getCollectionDAO).thenReturn(collectionDAO); - - MigrationValidationClient migrationClient = mock(MigrationValidationClient.class); - migrationMock.when(MigrationValidationClient::getInstance).thenReturn(migrationClient); - - searchRepository = mock(SearchRepository.class); - entityMock.when(Entity::getSearchRepository).thenReturn(searchRepository); - - systemRepository = new SystemRepository(); - } - - @AfterEach - void tearDown() { - entityMock.close(); - migrationMock.close(); - } - - @Test - void testAllRequiredFieldsPresentReturnsEmpty() { - IndexMapping dashboard = mock(IndexMapping.class); - when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("dashboard", dashboard)); - when(searchRepository.getIndexFieldNames(dashboard)) - .thenReturn(SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS); - - List inconsistent = - systemRepository.findIndexesWithMissingConsumerFields(searchRepository); - - assertTrue(inconsistent.isEmpty()); - } - - @Test - void testMissingFieldIsReportedWithEntityAndField() { - IndexMapping dashboard = mock(IndexMapping.class); - Set partial = new HashSet<>(SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS); - partial.remove("tier"); - when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("dashboard", dashboard)); - when(searchRepository.getIndexFieldNames(dashboard)).thenReturn(partial); - - List inconsistent = - systemRepository.findIndexesWithMissingConsumerFields(searchRepository); - - assertEquals(1, inconsistent.size()); - assertTrue(inconsistent.get(0).contains("dashboard")); - assertTrue(inconsistent.get(0).contains("tier")); - } - - @Test - void testEmptyLiveFieldsSkippedSoMissingIndexIsNotDoubleReported() { - IndexMapping dashboard = mock(IndexMapping.class); - when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("dashboard", dashboard)); - when(searchRepository.getIndexFieldNames(dashboard)).thenReturn(Set.of()); - - List inconsistent = - systemRepository.findIndexesWithMissingConsumerFields(searchRepository); - - assertTrue(inconsistent.isEmpty()); - } - - @Test - void testNonCanaryEntityIsIgnored() { - IndexMapping tag = mock(IndexMapping.class); - when(searchRepository.getEntityIndexMap()).thenReturn(Map.of("tag", tag)); - - List inconsistent = - systemRepository.findIndexesWithMissingConsumerFields(searchRepository); - - assertTrue(inconsistent.isEmpty()); - } -} From 93fd91ef959e83a7813416a25b3367d3943bb826 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:06:35 +0200 Subject: [PATCH 14/19] feat(search): report reindex-pending in /system/validate, drop canary field check Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/jdbi3/SystemRepository.java | 91 +++++-------------- 1 file changed, 23 insertions(+), 68 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java index 0fdbff88ec7c..bb7c8de152b0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java @@ -80,8 +80,8 @@ import org.openmetadata.service.logstorage.LogStorageInterface; import org.openmetadata.service.migration.MigrationValidationClient; import org.openmetadata.service.resources.settings.SettingsCache; +import org.openmetadata.service.search.IndexMappingVersionTracker; import org.openmetadata.service.search.IndexMappingVersionTracker.MappingDriftState; -import org.openmetadata.service.search.SearchConsumerFields; import org.openmetadata.service.search.SearchRepository; import org.openmetadata.service.search.vector.client.EmbeddingClient; import org.openmetadata.service.secrets.SecretsManager; @@ -113,7 +113,7 @@ public class SystemRepository { private static final String FAILED_TO_UPDATE_SETTINGS = "Failed to Update Settings {}"; public static final String INTERNAL_SERVER_ERROR_WITH_REASON = "Internal Server Error. Reason :"; private static final String VECTOR_EMBEDDING_INDEX_KEY = "vectorEmbedding"; - private static final String MAPPING_CONSISTENCY_VALIDATION_KEY = "Index Mapping Consistency"; + private static final String REINDEX_STATUS_VALIDATION_KEY = "Search Reindex Status"; private final SystemDAO dao; private final MigrationValidationClient migrationValidationClient; @@ -123,9 +123,9 @@ private enum ValidationStepDescription { PIPELINE_SERVICE_CLIENT("Validate that the pipeline service client is available."), JWT_TOKEN("Validate that the ingestion-bot JWT token can be properly decoded."), MIGRATION("Validate that all the necessary migrations have been properly executed."), - SEARCH_MAPPING( - "Validate that deployed search indexes expose the denormalized fields used by RBAC, " - + "Data Quality, Incidents, Lineage, and Data Insights."); + SEARCH_REINDEX( + "Validate that every deployed search index was built from the current code mapping " + + "(i.e. no reindex is pending)."); public final String key; @@ -564,8 +564,7 @@ public ValidationResponse validateSystem( "Semantic Search", getEmbeddingsValidation(applicationConfig)); } - validation.setAdditionalProperty( - MAPPING_CONSISTENCY_VALIDATION_KEY, getMappingConsistencyValidation()); + validation.setAdditionalProperty(REINDEX_STATUS_VALIDATION_KEY, getReindexStatusValidation()); addExtraValidations(applicationConfig, validation); @@ -894,9 +893,8 @@ List findMissingIndexes(SearchRepository searchRepository) { List missing = new ArrayList<>(); boolean semanticSearchEnabled = searchRepository.isVectorEmbeddingEnabled(); try { - Map indexMap = - searchRepository.getEntityIndexMap(); - for (Map.Entry entry : indexMap.entrySet()) { + Map indexMap = searchRepository.getEntityIndexMap(); + for (Map.Entry entry : indexMap.entrySet()) { if (!semanticSearchEnabled && VECTOR_EMBEDDING_INDEX_KEY.equals(entry.getKey())) { continue; } @@ -910,16 +908,16 @@ List findMissingIndexes(SearchRepository searchRepository) { return missing; } - private StepValidation getMappingConsistencyValidation() { + private StepValidation getReindexStatusValidation() { StepValidation step = - new StepValidation().withDescription(ValidationStepDescription.SEARCH_MAPPING.key); + new StepValidation().withDescription(ValidationStepDescription.SEARCH_REINDEX.key); SearchRepository searchRepository = Entity.getSearchRepository(); StepValidation result; if (searchRepository.getSearchClient().isClientAvailable()) { - List inconsistent = findIndexesWithMissingConsumerFields(searchRepository); + ReindexStatus status = computeReindexStatus(searchRepository); result = - step.withPassed(inconsistent.isEmpty()) - .withMessage(buildMappingConsistencyMessage(inconsistent)); + step.withPassed(status.stalePending().isEmpty()) + .withMessage(buildReindexStatusMessage(status)); } else { result = step.withPassed(Boolean.TRUE).withMessage("Skipped: search instance is not reachable."); @@ -927,62 +925,19 @@ private StepValidation getMappingConsistencyValidation() { return result; } - private String buildMappingConsistencyMessage(List inconsistent) { - String message; - if (inconsistent.isEmpty()) { - message = - "All core data asset indexes expose the fields used by RBAC, Data Quality, Incidents, " - + "Lineage, and Data Insights."; - } else { - message = - String.format( - "WARNING: %d index(es) are missing denormalized fields that RBAC, Data Quality, " - + "Incidents, Lineage, and Data Insights depend on — a reindex is required to " - + "restore them. Affected: %s", - inconsistent.size(), inconsistent); - } - return message; - } - - @VisibleForTesting - List findIndexesWithMissingConsumerFields(SearchRepository searchRepository) { - List inconsistent = new ArrayList<>(); + private ReindexStatus computeReindexStatus(SearchRepository searchRepository) { + ReindexStatus status = new ReindexStatus(new ArrayList<>(), 0); try { - Map indexMap = searchRepository.getEntityIndexMap(); - for (String entityType : SearchConsumerFields.CANARY_DATA_ASSET_ENTITIES) { - collectMissingConsumerFields( - searchRepository, entityType, indexMap.get(entityType), inconsistent); - } + String version = System.getProperty("project.version", "1.8.0-SNAPSHOT"); + IndexMappingVersionTracker tracker = + new IndexMappingVersionTracker(Entity.getCollectionDAO(), version, "system"); + Set existingIndexes = new HashSet<>(searchRepository.getEntityIndexMap().keySet()); + existingIndexes.removeAll(findMissingIndexes(searchRepository)); + status = classifyReindexStatus(tracker.computeDrift(), existingIndexes); } catch (Exception e) { - LOG.warn("Failed to validate index mapping consistency: {}", e.getMessage()); + LOG.warn("Failed to compute search reindex status: {}", e.getMessage()); } - return inconsistent; - } - - private void collectMissingConsumerFields( - SearchRepository searchRepository, - String entityType, - IndexMapping indexMapping, - List inconsistent) { - if (indexMapping != null) { - Set liveFields = searchRepository.getIndexFieldNames(indexMapping); - if (!liveFields.isEmpty()) { - List missing = missingConsumerFields(liveFields); - if (!missing.isEmpty()) { - inconsistent.add(entityType + " (missing: " + missing + ")"); - } - } - } - } - - private List missingConsumerFields(Set liveFields) { - List missing = new ArrayList<>(); - for (String required : SearchConsumerFields.CANARY_REQUIRED_TOP_LEVEL_FIELDS) { - if (!liveFields.contains(required)) { - missing.add(required); - } - } - return missing; + return status; } private boolean validateDataInsights() { From 981bebc20f5620e45c6e43672e252fb06acc5b21 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:20:27 +0200 Subject: [PATCH 15/19] feat(search): stamp mapping versions after createIndexes recreate Track which entity indexes were successfully recreated in createIndexes() and stamp their mapping versions via IndexMappingVersionTracker, skipping any that threw during finalizeReindex. Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/search/SearchRepository.java | 16 ++++++- .../search/SearchRepositoryBehaviorTest.java | 46 +++++++++++++++++-- 2 files changed, 57 insertions(+), 5 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index 00d45368f1c4..ff4f9e7b6596 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -308,6 +308,7 @@ public void createIndexes() { RecreateIndexHandler recreateIndexHandler = this.createReindexHandler(); ReindexContext context = recreateIndexHandler.reCreateIndexes(entityIndexMap.keySet()); if (context != null) { + List recreated = new ArrayList<>(); for (String entityType : context.getEntities()) { try { String originalIndex = context.getOriginalIndex(entityType).orElse(null); @@ -331,11 +332,24 @@ public void createIndexes() { .parentAliases(parentAliases) .build(); recreateIndexHandler.finalizeReindex(entityReindexContext, true); - + recreated.add(entityType); } catch (Exception ex) { LOG.error("Failed to recreate index for entity {}", entityType, ex); } } + stampRecreatedMappings(recreated); + } + } + + protected void stampRecreatedMappings(List entityTypes) { + if (!nullOrEmpty(entityTypes)) { + try { + String version = System.getProperty("project.version", "1.8.0-SNAPSHOT"); + new IndexMappingVersionTracker(Entity.getCollectionDAO(), version, "system") + .updateMappingVersions(entityTypes); + } catch (Exception e) { + LOG.warn("Failed to stamp index mapping versions after createIndexes: {}", e.getMessage()); + } } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java index 157b66db76ce..9f22d43e6c76 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/SearchRepositoryBehaviorTest.java @@ -30,6 +30,7 @@ import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Optional; import java.util.Set; import java.util.UUID; import org.apache.commons.lang3.tuple.Pair; @@ -2116,13 +2117,13 @@ void createIndexesFinalizesEachRecreatedEntityAndContinuesOnFailure() { when(context.getEntities()) .thenReturn(new LinkedHashSet<>(List.of(Entity.TABLE, Entity.DOMAIN))); when(context.getOriginalIndex(any())) - .thenAnswer(invocation -> java.util.Optional.of("original_" + invocation.getArgument(0))); + .thenAnswer(invocation -> Optional.of("original_" + invocation.getArgument(0))); when(context.getCanonicalIndex(any())) - .thenAnswer(invocation -> java.util.Optional.of("canonical_" + invocation.getArgument(0))); + .thenAnswer(invocation -> Optional.of("canonical_" + invocation.getArgument(0))); when(context.getStagedIndex(any())) - .thenAnswer(invocation -> java.util.Optional.of("staged_" + invocation.getArgument(0))); + .thenAnswer(invocation -> Optional.of("staged_" + invocation.getArgument(0))); when(context.getCanonicalAlias(any())) - .thenAnswer(invocation -> java.util.Optional.of("alias_" + invocation.getArgument(0))); + .thenAnswer(invocation -> Optional.of("alias_" + invocation.getArgument(0))); when(context.getExistingAliases(any())) .thenAnswer(invocation -> Set.of("existing_" + invocation.getArgument(0))); when(context.getParentAliases(any())) @@ -2148,6 +2149,43 @@ void createIndexesFinalizesEachRecreatedEntityAndContinuesOnFailure() { assertEquals(Entity.DOMAIN, contextCaptor.getAllValues().get(1).getEntityType()); } + @Test + void createIndexesStampsOnlySucceededEntities() throws Exception { + SearchRepository spyRepository = spy(repository); + RecreateIndexHandler recreateIndexHandler = mock(RecreateIndexHandler.class); + ReindexContext context = mock(ReindexContext.class); + + doReturn(recreateIndexHandler).when(spyRepository).createReindexHandler(); + when(recreateIndexHandler.reCreateIndexes(any())).thenReturn(context); + when(context.getEntities()) + .thenReturn(new LinkedHashSet<>(List.of(Entity.TABLE, Entity.DOMAIN))); + when(context.getOriginalIndex(any())) + .thenAnswer(invocation -> Optional.of("original_" + invocation.getArgument(0))); + when(context.getCanonicalIndex(any())) + .thenAnswer(invocation -> Optional.of("canonical_" + invocation.getArgument(0))); + when(context.getStagedIndex(any())) + .thenAnswer(invocation -> Optional.of("staged_" + invocation.getArgument(0))); + when(context.getCanonicalAlias(any())) + .thenAnswer(invocation -> Optional.of("alias_" + invocation.getArgument(0))); + when(context.getExistingAliases(any())) + .thenAnswer(invocation -> Set.of("existing_" + invocation.getArgument(0))); + when(context.getParentAliases(any())) + .thenAnswer(invocation -> List.of("parent_" + invocation.getArgument(0))); + + doNothing().when(spyRepository).stampRecreatedMappings(any()); + doNothing() + .doThrow(new RuntimeException("boom")) + .when(recreateIndexHandler) + .finalizeReindex(any(EntityReindexContext.class), eq(true)); + + spyRepository.createIndexes(); + + @SuppressWarnings("unchecked") + ArgumentCaptor> stampCaptor = ArgumentCaptor.forClass(List.class); + verify(spyRepository).stampRecreatedMappings(stampCaptor.capture()); + assertEquals(List.of(Entity.TABLE), stampCaptor.getValue()); + } + @Test void deleteEntityIndexRemovesDomainReferencesAndChildren() throws Exception { EntityInterface domain = mockEntity(Entity.DOMAIN, UUID.randomUUID(), "finance"); From 32e60fb86a06e1566272b0bbb1f96d695dc212cd Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:28:44 +0200 Subject: [PATCH 16/19] feat(search): stamp mapping versions on successful reindex (UI/scheduled/CLI) Co-Authored-By: Claude Sonnet 4.6 --- .../bundles/searchIndex/SearchIndexApp.java | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java index 7122574a3c97..89a8af8d418e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java @@ -1,8 +1,12 @@ package org.openmetadata.service.apps.bundles.searchIndex; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; + import jakarta.ws.rs.core.Response; +import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.openmetadata.schema.entity.app.App; @@ -15,6 +19,7 @@ import org.openmetadata.service.exception.AppException; import org.openmetadata.service.jdbi3.AppRepository; import org.openmetadata.service.jdbi3.CollectionDAO; +import org.openmetadata.service.search.IndexMappingVersionTracker; import org.openmetadata.service.search.SearchRepository; import org.quartz.JobExecutionContext; @@ -63,6 +68,24 @@ public void execute(JobExecutionContext ctx) { this.orchestrator = orch; orch.run(jobData); this.jobData = orch.getJobData(); + stampReindexedMappings(this.jobData); + } + + private void stampReindexedMappings(EventPublisherJob job) { + if (job != null && job.getStatus() == EventPublisherJob.Status.COMPLETED) { + Set targeted = job.getEntities(); + Collection toStamp = + (nullOrEmpty(targeted) || targeted.contains(SearchIndexEntityTypes.ALL)) + ? searchRepository.getEntityIndexMap().keySet() + : targeted; + try { + String version = System.getProperty("project.version", "1.8.0-SNAPSHOT"); + new IndexMappingVersionTracker(collectionDAO, version, "system") + .updateMappingVersions(toStamp); + } catch (Exception e) { + LOG.warn("Failed to stamp index mapping versions after reindex", e); + } + } } @Override From e0c2397c138895cac79b4a05653ba4ca188ac7fb Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:33:53 +0200 Subject: [PATCH 17/19] refactor(search): drop redundant CLI mapping-version stamp (SearchIndexApp owns it) Co-Authored-By: Claude Opus 4.8 (1M context) --- .../service/util/OpenMetadataOperations.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java index 352d75497199..8fe2a2076652 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/util/OpenMetadataOperations.java @@ -1790,7 +1790,6 @@ private int executeSearchReindexApp( // Check for index mapping changes only when running from CLI IndexMappingVersionTracker versionTracker = null; - boolean shouldUpdateVersions = false; ReindexingProgressMonitor progressMonitor = null; boolean shouldReindex = true; @@ -1820,8 +1819,6 @@ private int executeSearchReindexApp( } } } else { - shouldUpdateVersions = true; - // If 'all' entities were requested, only reindex changed ones if (entities.contains("all")) { entities = new HashSet<>(changedMappings); @@ -1833,7 +1830,6 @@ private int executeSearchReindexApp( LOG.info( "✅ Smart reindexing: None of the requested entities have mapping changes, skipping reindex"); shouldReindex = false; - shouldUpdateVersions = false; // Send Slack notification if configured if (slackBotToken != null @@ -1917,18 +1913,6 @@ private int executeSearchReindexApp( int result = waitAndReturnReindexingAppStatus(app, currentTime, progressMonitor); - // Update mapping versions after successful reindexing - if (result == 0 && shouldUpdateVersions && versionTracker != null) { - try { - versionTracker.updateMappingVersions(); - LOG.info( - "✅ Smart reindexing: Updated mapping versions in database for future change detection"); - } catch (Exception e) { - LOG.warn("⚠️ Failed to update index mapping versions in database", e); - // Don't fail the operation if version update fails - } - } - return result; } From 0607357900ea1f4960e5abf97bdb234edad06fca Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:39:08 +0200 Subject: [PATCH 18/19] chore(search): remove orphaned getIndexFieldNames after canary check removal Co-Authored-By: Claude Sonnet 4.6 --- .../service/search/IndexManagementClient.java | 13 ---------- .../service/search/SearchRepository.java | 5 ---- .../elasticsearch/ElasticSearchClient.java | 5 ---- .../ElasticSearchIndexManager.java | 24 ------------------- .../search/opensearch/OpenSearchClient.java | 5 ---- .../opensearch/OpenSearchIndexManager.java | 24 ------------------- 6 files changed, 76 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java index 53ffa040f0c5..64fe48045c51 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/IndexManagementClient.java @@ -207,17 +207,4 @@ default long getDocumentCount(String indexName) { } return 0; } - - /** - * Get the top-level field (property) names declared in an index's live mapping. Used by the - * "Index Mapping Consistency" health check to detect deployed indexes missing the denormalized - * fields product features rely on (e.g. left stale after an upgrade without a reindex). - * - * @param indexName the index (or alias) name to inspect - * @return the set of top-level field names, or an empty set if the index is unreachable/missing - */ - default Set getIndexFieldNames(String indexName) { - throw new UnsupportedOperationException( - "getIndexFieldNames is not implemented for this search client"); - } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index ff4f9e7b6596..4ccb3ccf21a9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -748,11 +748,6 @@ public boolean indexExists(IndexMapping indexMapping) { return !searchClient.getIndicesByAlias(indexName).isEmpty(); } - public Set getIndexFieldNames(IndexMapping indexMapping) { - String indexName = indexMapping.getIndexName(clusterAlias); - return searchClient.getIndexFieldNames(indexName); - } - public void createIndex(IndexMapping indexMapping) { try { String indexName = indexMapping.getIndexName(clusterAlias); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java index 05312003b883..abb2d2d0050a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchClient.java @@ -217,11 +217,6 @@ public boolean indexExists(String indexName) { return indexManager.indexExists(indexName); } - @Override - public Set getIndexFieldNames(String indexName) { - return indexManager.getIndexFieldNames(indexName); - } - @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { indexManager.createIndex(indexMapping, indexMappingContent); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java index 4f3ce58d21fb..0f32214bf507 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticSearchIndexManager.java @@ -11,7 +11,6 @@ import es.co.elastic.clients.elasticsearch.indices.ForcemergeResponse; import es.co.elastic.clients.elasticsearch.indices.GetAliasRequest; import es.co.elastic.clients.elasticsearch.indices.GetAliasResponse; -import es.co.elastic.clients.elasticsearch.indices.GetMappingResponse; import es.co.elastic.clients.elasticsearch.indices.PutIndicesSettingsRequest; import es.co.elastic.clients.elasticsearch.indices.PutIndicesSettingsResponse; import es.co.elastic.clients.elasticsearch.indices.PutMappingRequest; @@ -64,29 +63,6 @@ public boolean indexExists(String indexName) { } } - @Override - public Set getIndexFieldNames(String indexName) { - Set fieldNames = new HashSet<>(); - if (isClientAvailable) { - try { - GetMappingResponse response = client.indices().getMapping(g -> g.index(indexName)); - response - .mappings() - .values() - .forEach( - mappingRecord -> { - if (mappingRecord.mappings() != null - && mappingRecord.mappings().properties() != null) { - fieldNames.addAll(mappingRecord.mappings().properties().keySet()); - } - }); - } catch (Exception e) { - LOG.warn("Failed to get field names for index {}: {}", indexName, e.getMessage()); - } - } - return fieldNames; - } - @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { if (!isClientAvailable) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java index e251df36f48e..930852c59119 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchClient.java @@ -204,11 +204,6 @@ public boolean indexExists(String indexName) { return indexManager.indexExists(indexName); } - @Override - public Set getIndexFieldNames(String indexName) { - return indexManager.getIndexFieldNames(indexName); - } - @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { indexManager.createIndex(indexMapping, indexMappingContent); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java index ed3e5d797484..629d0121c996 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchIndexManager.java @@ -25,7 +25,6 @@ import os.org.opensearch.client.opensearch.indices.ForcemergeResponse; import os.org.opensearch.client.opensearch.indices.GetAliasRequest; import os.org.opensearch.client.opensearch.indices.GetAliasResponse; -import os.org.opensearch.client.opensearch.indices.GetMappingResponse; import os.org.opensearch.client.opensearch.indices.IndexSettings; import os.org.opensearch.client.opensearch.indices.PutIndicesSettingsRequest; import os.org.opensearch.client.opensearch.indices.PutIndicesSettingsResponse; @@ -67,29 +66,6 @@ public boolean indexExists(String indexName) { } } - @Override - public Set getIndexFieldNames(String indexName) { - Set fieldNames = new HashSet<>(); - if (isClientAvailable) { - try { - GetMappingResponse response = client.indices().getMapping(g -> g.index(indexName)); - response - .result() - .values() - .forEach( - mappingRecord -> { - if (mappingRecord.mappings() != null - && mappingRecord.mappings().properties() != null) { - fieldNames.addAll(mappingRecord.mappings().properties().keySet()); - } - }); - } catch (Exception e) { - LOG.warn("Failed to get field names for index {}: {}", indexName, e.getMessage()); - } - } - return fieldNames; - } - @Override public void createIndex(IndexMapping indexMapping, String indexMappingContent) { if (!isClientAvailable) { From 31558655f591d0b6415cb4e506293376303a23f8 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Mon, 8 Jun 2026 17:55:16 +0200 Subject: [PATCH 19/19] docs(search): update SearchConsumerFields javadoc after canary check removal --- .../service/search/SearchConsumerFields.java | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java index 26f0cc6b0250..b80fa2b8a286 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchConsumerFields.java @@ -22,13 +22,10 @@ * breaks those consumers (empty facets, wrong access-control results, zeroed Data Quality widgets) * with no compile-time or boot-time failure. * - *

Two guards reference this class so the contract lives in one place: - * - *

    - *
  • {@code SearchConsumerFieldContractTest} fails CI when a shipped mapping file violates it. - *
  • {@code SystemRepository}'s "Index Mapping Consistency" health check warns when a deployed - * index is missing these fields (e.g. left stale after an upgrade without a reindex). - *
+ *

{@code SearchConsumerFieldContractTest} references this class so the contract lives in one + * place: it fails CI when a shipped mapping file renames, retypes, or drops one of these fields. + * (Deployed-index staleness is covered separately by {@code SystemRepository}'s "Search Reindex + * Status" check, which compares mapping hashes rather than these specific fields.) */ public final class SearchConsumerFields { private SearchConsumerFields() {}