diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/QueryFilterBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/QueryFilterBuilder.java index 471c5a16f89b..39bfc96474f9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/QueryFilterBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/QueryFilterBuilder.java @@ -204,6 +204,7 @@ private static void addNestedTermCondition( ObjectNode nestedNode = MAPPER.createObjectNode(); ObjectNode nestedInner = nestedNode.putObject("nested"); nestedInner.put("path", path); + nestedInner.put("ignore_unmapped", true); ObjectNode termNode = MAPPER.createObjectNode(); termNode.putObject(TERM_KEY).put(fieldPath, fieldValue); nestedInner.set(QUERY_KEY, termNode); @@ -215,6 +216,7 @@ private static void addNestedMatchCondition( ObjectNode nestedNode = MAPPER.createObjectNode(); ObjectNode nestedInner = nestedNode.putObject("nested"); nestedInner.put("path", path); + nestedInner.put("ignore_unmapped", true); ObjectNode matchNode = MAPPER.createObjectNode(); matchNode.putObject(MATCH_KEY).put(fieldPath, fieldValue); nestedInner.set(QUERY_KEY, matchNode); @@ -226,6 +228,7 @@ private static void addNestedOrCondition( ObjectNode nestedNode = MAPPER.createObjectNode(); ObjectNode nestedInner = nestedNode.putObject("nested"); nestedInner.put("path", path); + nestedInner.put("ignore_unmapped", true); ObjectNode orCondition = MAPPER.createObjectNode(); ObjectNode innerBool = orCondition.putObject(BOOL_KEY); ArrayNode shouldArray = innerBool.putArray(SHOULD_KEY); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index 9ca9cd242b85..f5313a5d9151 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -154,7 +154,7 @@ private String getOwnerCondition() { String ownersList = Arrays.stream(owners.split(",")).collect(Collectors.joining("\", \"", "\"", "\"")); return String.format( - "{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[%s]}}}}", + "{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[%s]}},\"ignore_unmapped\":true}}", ownersList); } return ""; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticQueryBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticQueryBuilder.java index c30c980ffbaa..543e75d6363a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticQueryBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/ElasticQueryBuilder.java @@ -272,7 +272,7 @@ public static Query existsQuery(String field) { } public static Query nestedQuery(String path, Query query) { - return Query.of(q -> q.nested(n -> n.path(path).query(query))); + return Query.of(q -> q.nested(n -> n.path(path).query(query).ignoreUnmapped(true))); } public static Query functionScoreQuery( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/queries/ElasticQueryBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/queries/ElasticQueryBuilder.java index ae44038d29d6..6324602509cd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/queries/ElasticQueryBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/elasticsearch/queries/ElasticQueryBuilder.java @@ -172,7 +172,7 @@ public ElasticQueryBuilder existsQuery(String field) { public ElasticQueryBuilder nestedQuery(String path, OMQueryBuilder innerQuery) { Query inner = ((ElasticQueryBuilder) innerQuery).build(); - this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner))); + this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner).ignoreUnmapped(true))); return this; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchQueryBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchQueryBuilder.java index a47a42b637ba..bbe429084cd8 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchQueryBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/OpenSearchQueryBuilder.java @@ -271,7 +271,7 @@ public static Query existsQuery(String field) { } public static Query nestedQuery(String path, Query query) { - return Query.of(q -> q.nested(n -> n.path(path).query(query))); + return Query.of(q -> q.nested(n -> n.path(path).query(query).ignoreUnmapped(true))); } public static Query functionScoreQuery( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/queries/OpenSearchQueryBuilder.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/queries/OpenSearchQueryBuilder.java index b6cb98971fd0..a5910b379116 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/queries/OpenSearchQueryBuilder.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/opensearch/queries/OpenSearchQueryBuilder.java @@ -175,7 +175,7 @@ public OpenSearchQueryBuilder existsQuery(String field) { public OpenSearchQueryBuilder nestedQuery(String path, OMQueryBuilder innerQuery) { Query inner = ((OpenSearchQueryBuilder) innerQuery).build(); - this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner))); + this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner).ignoreUnmapped(true))); return this; } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java index 32ded21d37e6..f420fe8968a3 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/SearchListFilterTest.java @@ -104,6 +104,9 @@ void testGetFilterQueryReturnsOnlyQuerySection() { assertEquals("owners", actual.at("/bool/filter/1/nested/path").asText()); assertEquals("owner1", actual.at("/bool/filter/1/nested/query/terms/owners.id/0").asText()); assertEquals("owner2", actual.at("/bool/filter/1/nested/query/terms/owners.id/1").asText()); + assertTrue( + actual.at("/bool/filter/1/nested/ignore_unmapped").asBoolean(), + "Nested owners query must set ignore_unmapped to true"); } @Test @@ -139,7 +142,7 @@ void testGenericFiltersCombineDomainOwnersAndCreatedBy() { actual.contains("{\"term\": {\"domains.fullyQualifiedName\": \"finance.\\\"raw\\\"\"}}")); assertTrue( actual.contains( - "{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[\"owner1\", \"owner2\"]}}}}")); + "{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[\"owner1\", \"owner2\"]}},\"ignore_unmapped\":true}}")); assertTrue(actual.contains("{\"term\": {\"createdBy\": \"user\\\"name\"}}")); } @@ -387,6 +390,41 @@ void testResolutionStatusCondition_withTestCaseFqn() { "Expected testCaseFqn filter but got: " + actual); } + @Test + void testNestedQueryWorksForMappedFields() { + // Verifies nested query produces correct structure for indexes with owners as nested + SearchListFilter searchListFilter = new SearchListFilter(); + searchListFilter.addQueryParam("owners", "owner-abc,owner-def"); + + JsonNode actual = parse(searchListFilter.getFilterQuery(null)); + + JsonNode nested = actual.at("/bool/filter/1/nested"); + assertFalse(nested.isMissingNode(), "nested query must be present"); + assertEquals("owners", nested.at("/path").asText(), "must target correct nested path"); + assertEquals( + "owner-abc", + nested.at("/query/terms/owners.id/0").asText(), + "must contain first owner value"); + assertEquals( + "owner-def", + nested.at("/query/terms/owners.id/1").asText(), + "must contain second owner value"); + } + + @Test + void testNestedQueryDoesNotFailForUnmappedFields() { + // Verifies ignore_unmapped is set so indexes without owners (e.g. user) don't throw errors + SearchListFilter searchListFilter = new SearchListFilter(); + searchListFilter.addQueryParam("owners", "owner-abc"); + + JsonNode actual = parse(searchListFilter.getFilterQuery(null)); + + JsonNode nested = actual.at("/bool/filter/1/nested"); + assertTrue( + nested.at("/ignore_unmapped").asBoolean(), + "must set ignore_unmapped so query works on indexes without this nested field"); + } + private JsonNode parse(String json) { return JsonUtils.readTree(json); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/QueryFilterBuilderTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/QueryFilterBuilderTest.java index 41af8b35a92c..bd846bb7fbe1 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/QueryFilterBuilderTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/QueryFilterBuilderTest.java @@ -52,6 +52,9 @@ void buildTeamAssetsCountFilterTargetsTeamOwnersAndNonDeletedAssets() { assertEquals("owners", filter.at("/query/bool/must/0/nested/path").asText()); assertEquals("team", filter.at("/query/bool/must/0/nested/query/term/owners.type").asText()); + assertTrue( + filter.at("/query/bool/must/0/nested/ignore_unmapped").asBoolean(), + "Nested owners query must set ignore_unmapped to true"); assertFalse(filter.at("/query/bool/must/1/term/deleted").asBoolean()); } @@ -68,6 +71,9 @@ void buildOwnerAssetsFilterUsesNestedMatchAndEntityTypeFilter() { assertEquals("owners", filter.at("/query/bool/must/0/nested/path").asText()); assertEquals("team-id", filter.at("/query/bool/must/0/nested/query/match/owners.id").asText()); + assertTrue( + filter.at("/query/bool/must/0/nested/ignore_unmapped").asBoolean(), + "Nested owners query must set ignore_unmapped to true"); assertFalse(filter.at("/query/bool/must/1/term/deleted").asBoolean()); assertEquals(Entity.TABLE, filter.at("/query/bool/must/2/term/entityType").asText()); } @@ -132,6 +138,9 @@ void buildUserAssetsFilterAddsNestedOrConditionsForAllOwners() { JsonNode filter = parse(QueryFilterBuilder.buildUserAssetsFilter(query)); assertEquals("owners", filter.at("/query/bool/must/0/nested/path").asText()); + assertTrue( + filter.at("/query/bool/must/0/nested/ignore_unmapped").asBoolean(), + "Nested owners query must set ignore_unmapped to true"); assertEquals( "user-id", filter.at("/query/bool/must/0/nested/query/bool/should/0/term/owners.id").asText()); @@ -141,6 +150,89 @@ void buildUserAssetsFilterAddsNestedOrConditionsForAllOwners() { assertFalse(filter.at("/query/bool/must/1/term/deleted").asBoolean()); } + @Test + void nestedTermConditionWorksForMappedFields() { + // Verifies nested term query produces correct structure for indexes with the nested field + JsonNode filter = parse(QueryFilterBuilder.buildTeamAssetsCountFilter()); + + JsonNode nested = filter.at("/query/bool/must/0/nested"); + assertEquals("owners", nested.at("/path").asText(), "must target correct nested path"); + assertFalse(nested.at("/query").isMissingNode(), "must contain inner query"); + assertEquals( + "team", nested.at("/query/term/owners.type").asText(), "must filter on term value"); + } + + @Test + void nestedTermConditionDoesNotFailForUnmappedFields() { + // Verifies ignore_unmapped is set so indexes without the nested field don't throw errors + JsonNode filter = parse(QueryFilterBuilder.buildTeamAssetsCountFilter()); + + JsonNode nested = filter.at("/query/bool/must/0/nested"); + assertTrue( + nested.at("/ignore_unmapped").asBoolean(), + "must set ignore_unmapped so query works on indexes without this nested field"); + } + + @Test + void nestedMatchConditionWorksForMappedFields() { + InheritedFieldQuery query = + InheritedFieldQuery.builder().fieldPath("owners.id").fieldValue("owner-123").build(); + + JsonNode filter = parse(QueryFilterBuilder.buildOwnerAssetsFilter(query)); + + JsonNode nested = filter.at("/query/bool/must/0/nested"); + assertEquals("owners", nested.at("/path").asText(), "must target correct nested path"); + assertEquals( + "owner-123", + nested.at("/query/match/owners.id").asText(), + "must match on the provided value"); + } + + @Test + void nestedMatchConditionDoesNotFailForUnmappedFields() { + InheritedFieldQuery query = + InheritedFieldQuery.builder().fieldPath("owners.id").fieldValue("owner-123").build(); + + JsonNode filter = parse(QueryFilterBuilder.buildOwnerAssetsFilter(query)); + + JsonNode nested = filter.at("/query/bool/must/0/nested"); + assertTrue( + nested.at("/ignore_unmapped").asBoolean(), + "must set ignore_unmapped so query works on indexes without this nested field"); + } + + @Test + void nestedOrConditionWorksForMappedFields() { + InheritedFieldQuery query = + InheritedFieldQuery.builder() + .fieldPath("owners.id") + .fieldValues(List.of("id-1", "id-2", "id-3")) + .build(); + + JsonNode filter = parse(QueryFilterBuilder.buildUserAssetsFilter(query)); + + JsonNode nested = filter.at("/query/bool/must/0/nested"); + assertEquals("owners", nested.at("/path").asText(), "must target correct nested path"); + assertEquals( + 3, nested.at("/query/bool/should").size(), "must contain all values in should clause"); + } + + @Test + void nestedOrConditionDoesNotFailForUnmappedFields() { + InheritedFieldQuery query = + InheritedFieldQuery.builder() + .fieldPath("owners.id") + .fieldValues(List.of("id-1", "id-2")) + .build(); + + JsonNode filter = parse(QueryFilterBuilder.buildUserAssetsFilter(query)); + + JsonNode nested = filter.at("/query/bool/must/0/nested"); + assertTrue( + nested.at("/ignore_unmapped").asBoolean(), + "must set ignore_unmapped so query works on indexes without this nested field"); + } + private JsonNode parse(String json) { return JsonUtils.readTree(json); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java index acae0d4bc38a..aba229004f7f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/ElasticSearchRBACConditionEvaluatorTest.java @@ -1563,4 +1563,62 @@ void testDenyOnTableIncludesChildAliasesInIndexFilter() { "$.bool.must_not[*].bool.must[?(@.terms._index[?(@ == 'column')])]", "Deny policy should include 'column' (child of table) in must_not _index filter"); } + + @Test + void testNestedQueryWorksForMappedFields() { + // Verifies nested query produces correct structure for indexes that have the field mapped + ElasticQueryBuilderFactory factory = new ElasticQueryBuilderFactory(); + + OMQueryBuilder nested = + factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc")); + Query query = ((ElasticQueryBuilder) nested).build(); + String json = serializeQueryToJson(query); + + assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path"); + assertTrue(json.contains("\"owners.id\""), "must query the nested field"); + assertTrue(json.contains("\"user-abc\""), "must contain the search value"); + } + + @Test + void testNestedQueryDoesNotFailForUnmappedFields() { + // Verifies ignore_unmapped=true is set so indexes without the nested field don't throw errors + ElasticQueryBuilderFactory factory = new ElasticQueryBuilderFactory(); + + OMQueryBuilder nested = + factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc")); + Query query = ((ElasticQueryBuilder) nested).build(); + String json = serializeQueryToJson(query); + + assertTrue( + json.contains("\"ignore_unmapped\":true"), + "must set ignore_unmapped so query works on indexes without this nested field"); + } + + @Test + void testStaticNestedQueryWorksForMappedFields() { + Query inner = Query.of(q -> q.term(t -> t.field("owners.id").value("team-1"))); + + Query nested = + org.openmetadata.service.search.elasticsearch.ElasticQueryBuilder.nestedQuery( + "owners", inner); + String json = serializeQueryToJson(nested); + + assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path"); + assertTrue(json.contains("\"owners.id\""), "must query the nested field"); + assertTrue(json.contains("\"team-1\""), "must contain the search value"); + } + + @Test + void testStaticNestedQueryDoesNotFailForUnmappedFields() { + Query inner = Query.of(q -> q.term(t -> t.field("owners.id").value("team-1"))); + + Query nested = + org.openmetadata.service.search.elasticsearch.ElasticQueryBuilder.nestedQuery( + "owners", inner); + String json = serializeQueryToJson(nested); + + assertTrue( + json.contains("\"ignore_unmapped\":true"), + "must set ignore_unmapped so query works on indexes without this nested field"); + } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java index b819457ac82b..717cff9a86fe 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/search/security/OpenSearchRBACConditionEvaluatorTest.java @@ -370,4 +370,64 @@ void testHasAnyRoleWithNoMatchingInheritedRole() { "Query should result in match_nothing since user doesn't have Admin role"); } } + + @Test + void testNestedQueryWorksForMappedFields() { + // Verifies nested query produces correct structure for indexes that have the field mapped + OpenSearchQueryBuilderFactory factory = new OpenSearchQueryBuilderFactory(); + + OMQueryBuilder nested = + factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc")); + Query query = ((OpenSearchQueryBuilder) nested).build(); + String json = query.toJsonString(); + + assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path"); + assertTrue(json.contains("\"owners.id\""), "must query the nested field"); + assertTrue(json.contains("\"user-abc\""), "must contain the search value"); + } + + @Test + void testNestedQueryDoesNotFailForUnmappedFields() { + // Verifies ignore_unmapped=true is set so indexes without the nested field don't throw errors + OpenSearchQueryBuilderFactory factory = new OpenSearchQueryBuilderFactory(); + + OMQueryBuilder nested = + factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc")); + Query query = ((OpenSearchQueryBuilder) nested).build(); + String json = query.toJsonString(); + + assertTrue( + json.contains("\"ignore_unmapped\":true"), + "must set ignore_unmapped so query works on indexes without this nested field"); + } + + @Test + void testStaticNestedQueryWorksForMappedFields() { + Query inner = + Query.of(q -> q.term(t -> t.field("owners.id").value(v -> v.stringValue("team-1")))); + + Query nested = + org.openmetadata.service.search.opensearch.OpenSearchQueryBuilder.nestedQuery( + "owners", inner); + String json = nested.toJsonString(); + + assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path"); + assertTrue(json.contains("\"owners.id\""), "must query the nested field"); + assertTrue(json.contains("\"team-1\""), "must contain the search value"); + } + + @Test + void testStaticNestedQueryDoesNotFailForUnmappedFields() { + Query inner = + Query.of(q -> q.term(t -> t.field("owners.id").value(v -> v.stringValue("team-1")))); + + Query nested = + org.openmetadata.service.search.opensearch.OpenSearchQueryBuilder.nestedQuery( + "owners", inner); + String json = nested.toJsonString(); + + assertTrue( + json.contains("\"ignore_unmapped\":true"), + "must set ignore_unmapped so query works on indexes without this nested field"); + } }