Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -204,6 +204,7 @@ private static void addNestedTermCondition(
ObjectNode nestedNode = MAPPER.createObjectNode();
ObjectNode nestedInner = nestedNode.putObject("nested");
nestedInner.put("path", path);
nestedInner.put("ignore_unmapped", true);
ObjectNode termNode = MAPPER.createObjectNode();
termNode.putObject(TERM_KEY).put(fieldPath, fieldValue);
nestedInner.set(QUERY_KEY, termNode);
Expand All @@ -215,6 +216,7 @@ private static void addNestedMatchCondition(
ObjectNode nestedNode = MAPPER.createObjectNode();
ObjectNode nestedInner = nestedNode.putObject("nested");
nestedInner.put("path", path);
nestedInner.put("ignore_unmapped", true);
ObjectNode matchNode = MAPPER.createObjectNode();
matchNode.putObject(MATCH_KEY).put(fieldPath, fieldValue);
nestedInner.set(QUERY_KEY, matchNode);
Expand All @@ -226,6 +228,7 @@ private static void addNestedOrCondition(
ObjectNode nestedNode = MAPPER.createObjectNode();
ObjectNode nestedInner = nestedNode.putObject("nested");
nestedInner.put("path", path);
nestedInner.put("ignore_unmapped", true);
ObjectNode orCondition = MAPPER.createObjectNode();
ObjectNode innerBool = orCondition.putObject(BOOL_KEY);
ArrayNode shouldArray = innerBool.putArray(SHOULD_KEY);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -154,7 +154,7 @@ private String getOwnerCondition() {
String ownersList =
Arrays.stream(owners.split(",")).collect(Collectors.joining("\", \"", "\"", "\""));
return String.format(
"{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[%s]}}}}",
"{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[%s]}},\"ignore_unmapped\":true}}",
ownersList);
}
return "";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,7 +272,7 @@ public static Query existsQuery(String field) {
}

public static Query nestedQuery(String path, Query query) {
return Query.of(q -> q.nested(n -> n.path(path).query(query)));
return Query.of(q -> q.nested(n -> n.path(path).query(query).ignoreUnmapped(true)));
}

public static Query functionScoreQuery(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -172,7 +172,7 @@ public ElasticQueryBuilder existsQuery(String field) {

public ElasticQueryBuilder nestedQuery(String path, OMQueryBuilder innerQuery) {
Query inner = ((ElasticQueryBuilder) innerQuery).build();
this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner)));
this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner).ignoreUnmapped(true)));
return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -271,7 +271,7 @@ public static Query existsQuery(String field) {
}

public static Query nestedQuery(String path, Query query) {
return Query.of(q -> q.nested(n -> n.path(path).query(query)));
return Query.of(q -> q.nested(n -> n.path(path).query(query).ignoreUnmapped(true)));
}

public static Query functionScoreQuery(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,7 @@ public OpenSearchQueryBuilder existsQuery(String field) {

public OpenSearchQueryBuilder nestedQuery(String path, OMQueryBuilder innerQuery) {
Query inner = ((OpenSearchQueryBuilder) innerQuery).build();
this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner)));
this.query = Query.of(q -> q.nested(n -> n.path(path).query(inner).ignoreUnmapped(true)));
return this;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,9 @@ void testGetFilterQueryReturnsOnlyQuerySection() {
assertEquals("owners", actual.at("/bool/filter/1/nested/path").asText());
assertEquals("owner1", actual.at("/bool/filter/1/nested/query/terms/owners.id/0").asText());
assertEquals("owner2", actual.at("/bool/filter/1/nested/query/terms/owners.id/1").asText());
assertTrue(
actual.at("/bool/filter/1/nested/ignore_unmapped").asBoolean(),
"Nested owners query must set ignore_unmapped to true");
}

@Test
Expand Down Expand Up @@ -139,7 +142,7 @@ void testGenericFiltersCombineDomainOwnersAndCreatedBy() {
actual.contains("{\"term\": {\"domains.fullyQualifiedName\": \"finance.\\\"raw\\\"\"}}"));
assertTrue(
actual.contains(
"{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[\"owner1\", \"owner2\"]}}}}"));
"{\"nested\":{\"path\":\"owners\",\"query\":{\"terms\":{\"owners.id\":[\"owner1\", \"owner2\"]}},\"ignore_unmapped\":true}}"));
assertTrue(actual.contains("{\"term\": {\"createdBy\": \"user\\\"name\"}}"));
}

Expand Down Expand Up @@ -387,6 +390,41 @@ void testResolutionStatusCondition_withTestCaseFqn() {
"Expected testCaseFqn filter but got: " + actual);
}

@Test
void testNestedQueryWorksForMappedFields() {
// Verifies nested query produces correct structure for indexes with owners as nested
SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("owners", "owner-abc,owner-def");

JsonNode actual = parse(searchListFilter.getFilterQuery(null));

JsonNode nested = actual.at("/bool/filter/1/nested");
assertFalse(nested.isMissingNode(), "nested query must be present");
assertEquals("owners", nested.at("/path").asText(), "must target correct nested path");
assertEquals(
"owner-abc",
nested.at("/query/terms/owners.id/0").asText(),
"must contain first owner value");
assertEquals(
"owner-def",
nested.at("/query/terms/owners.id/1").asText(),
"must contain second owner value");
}

@Test
void testNestedQueryDoesNotFailForUnmappedFields() {
// Verifies ignore_unmapped is set so indexes without owners (e.g. user) don't throw errors
SearchListFilter searchListFilter = new SearchListFilter();
searchListFilter.addQueryParam("owners", "owner-abc");

JsonNode actual = parse(searchListFilter.getFilterQuery(null));

JsonNode nested = actual.at("/bool/filter/1/nested");
assertTrue(
nested.at("/ignore_unmapped").asBoolean(),
"must set ignore_unmapped so query works on indexes without this nested field");
}

private JsonNode parse(String json) {
return JsonUtils.readTree(json);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ void buildTeamAssetsCountFilterTargetsTeamOwnersAndNonDeletedAssets() {

assertEquals("owners", filter.at("/query/bool/must/0/nested/path").asText());
assertEquals("team", filter.at("/query/bool/must/0/nested/query/term/owners.type").asText());
assertTrue(
filter.at("/query/bool/must/0/nested/ignore_unmapped").asBoolean(),
"Nested owners query must set ignore_unmapped to true");
assertFalse(filter.at("/query/bool/must/1/term/deleted").asBoolean());
}

Expand All @@ -68,6 +71,9 @@ void buildOwnerAssetsFilterUsesNestedMatchAndEntityTypeFilter() {

assertEquals("owners", filter.at("/query/bool/must/0/nested/path").asText());
assertEquals("team-id", filter.at("/query/bool/must/0/nested/query/match/owners.id").asText());
assertTrue(
filter.at("/query/bool/must/0/nested/ignore_unmapped").asBoolean(),
"Nested owners query must set ignore_unmapped to true");
assertFalse(filter.at("/query/bool/must/1/term/deleted").asBoolean());
assertEquals(Entity.TABLE, filter.at("/query/bool/must/2/term/entityType").asText());
}
Expand Down Expand Up @@ -132,6 +138,9 @@ void buildUserAssetsFilterAddsNestedOrConditionsForAllOwners() {
JsonNode filter = parse(QueryFilterBuilder.buildUserAssetsFilter(query));

assertEquals("owners", filter.at("/query/bool/must/0/nested/path").asText());
assertTrue(
filter.at("/query/bool/must/0/nested/ignore_unmapped").asBoolean(),
"Nested owners query must set ignore_unmapped to true");
assertEquals(
"user-id",
filter.at("/query/bool/must/0/nested/query/bool/should/0/term/owners.id").asText());
Expand All @@ -141,6 +150,89 @@ void buildUserAssetsFilterAddsNestedOrConditionsForAllOwners() {
assertFalse(filter.at("/query/bool/must/1/term/deleted").asBoolean());
}

@Test
void nestedTermConditionWorksForMappedFields() {
// Verifies nested term query produces correct structure for indexes with the nested field
JsonNode filter = parse(QueryFilterBuilder.buildTeamAssetsCountFilter());

JsonNode nested = filter.at("/query/bool/must/0/nested");
assertEquals("owners", nested.at("/path").asText(), "must target correct nested path");
assertFalse(nested.at("/query").isMissingNode(), "must contain inner query");
assertEquals(
"team", nested.at("/query/term/owners.type").asText(), "must filter on term value");
}

@Test
void nestedTermConditionDoesNotFailForUnmappedFields() {
// Verifies ignore_unmapped is set so indexes without the nested field don't throw errors
JsonNode filter = parse(QueryFilterBuilder.buildTeamAssetsCountFilter());

JsonNode nested = filter.at("/query/bool/must/0/nested");
assertTrue(
nested.at("/ignore_unmapped").asBoolean(),
"must set ignore_unmapped so query works on indexes without this nested field");
}

@Test
void nestedMatchConditionWorksForMappedFields() {
InheritedFieldQuery query =
InheritedFieldQuery.builder().fieldPath("owners.id").fieldValue("owner-123").build();

JsonNode filter = parse(QueryFilterBuilder.buildOwnerAssetsFilter(query));

JsonNode nested = filter.at("/query/bool/must/0/nested");
assertEquals("owners", nested.at("/path").asText(), "must target correct nested path");
assertEquals(
"owner-123",
nested.at("/query/match/owners.id").asText(),
"must match on the provided value");
}

@Test
void nestedMatchConditionDoesNotFailForUnmappedFields() {
InheritedFieldQuery query =
InheritedFieldQuery.builder().fieldPath("owners.id").fieldValue("owner-123").build();

JsonNode filter = parse(QueryFilterBuilder.buildOwnerAssetsFilter(query));

JsonNode nested = filter.at("/query/bool/must/0/nested");
assertTrue(
nested.at("/ignore_unmapped").asBoolean(),
"must set ignore_unmapped so query works on indexes without this nested field");
}

@Test
void nestedOrConditionWorksForMappedFields() {
InheritedFieldQuery query =
InheritedFieldQuery.builder()
.fieldPath("owners.id")
.fieldValues(List.of("id-1", "id-2", "id-3"))
.build();

JsonNode filter = parse(QueryFilterBuilder.buildUserAssetsFilter(query));

JsonNode nested = filter.at("/query/bool/must/0/nested");
assertEquals("owners", nested.at("/path").asText(), "must target correct nested path");
assertEquals(
3, nested.at("/query/bool/should").size(), "must contain all values in should clause");
}

@Test
void nestedOrConditionDoesNotFailForUnmappedFields() {
InheritedFieldQuery query =
InheritedFieldQuery.builder()
.fieldPath("owners.id")
.fieldValues(List.of("id-1", "id-2"))
.build();

JsonNode filter = parse(QueryFilterBuilder.buildUserAssetsFilter(query));

JsonNode nested = filter.at("/query/bool/must/0/nested");
assertTrue(
nested.at("/ignore_unmapped").asBoolean(),
"must set ignore_unmapped so query works on indexes without this nested field");
}

private JsonNode parse(String json) {
return JsonUtils.readTree(json);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1563,4 +1563,62 @@ void testDenyOnTableIncludesChildAliasesInIndexFilter() {
"$.bool.must_not[*].bool.must[?(@.terms._index[?(@ == 'column')])]",
"Deny policy should include 'column' (child of table) in must_not _index filter");
}

@Test
void testNestedQueryWorksForMappedFields() {
// Verifies nested query produces correct structure for indexes that have the field mapped
ElasticQueryBuilderFactory factory = new ElasticQueryBuilderFactory();

OMQueryBuilder nested =
factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc"));
Query query = ((ElasticQueryBuilder) nested).build();
String json = serializeQueryToJson(query);

assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path");
assertTrue(json.contains("\"owners.id\""), "must query the nested field");
assertTrue(json.contains("\"user-abc\""), "must contain the search value");
}

@Test
void testNestedQueryDoesNotFailForUnmappedFields() {
// Verifies ignore_unmapped=true is set so indexes without the nested field don't throw errors
ElasticQueryBuilderFactory factory = new ElasticQueryBuilderFactory();

OMQueryBuilder nested =
factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc"));
Query query = ((ElasticQueryBuilder) nested).build();
String json = serializeQueryToJson(query);

assertTrue(
json.contains("\"ignore_unmapped\":true"),
"must set ignore_unmapped so query works on indexes without this nested field");
}

@Test
void testStaticNestedQueryWorksForMappedFields() {
Query inner = Query.of(q -> q.term(t -> t.field("owners.id").value("team-1")));

Query nested =
org.openmetadata.service.search.elasticsearch.ElasticQueryBuilder.nestedQuery(
"owners", inner);
String json = serializeQueryToJson(nested);

assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path");
assertTrue(json.contains("\"owners.id\""), "must query the nested field");
assertTrue(json.contains("\"team-1\""), "must contain the search value");
}

@Test
void testStaticNestedQueryDoesNotFailForUnmappedFields() {
Query inner = Query.of(q -> q.term(t -> t.field("owners.id").value("team-1")));

Query nested =
org.openmetadata.service.search.elasticsearch.ElasticQueryBuilder.nestedQuery(
"owners", inner);
String json = serializeQueryToJson(nested);

assertTrue(
json.contains("\"ignore_unmapped\":true"),
"must set ignore_unmapped so query works on indexes without this nested field");
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -370,4 +370,64 @@ void testHasAnyRoleWithNoMatchingInheritedRole() {
"Query should result in match_nothing since user doesn't have Admin role");
}
}

@Test
void testNestedQueryWorksForMappedFields() {
// Verifies nested query produces correct structure for indexes that have the field mapped
OpenSearchQueryBuilderFactory factory = new OpenSearchQueryBuilderFactory();

OMQueryBuilder nested =
factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc"));
Query query = ((OpenSearchQueryBuilder) nested).build();
String json = query.toJsonString();

assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path");
assertTrue(json.contains("\"owners.id\""), "must query the nested field");
assertTrue(json.contains("\"user-abc\""), "must contain the search value");
}

@Test
void testNestedQueryDoesNotFailForUnmappedFields() {
// Verifies ignore_unmapped=true is set so indexes without the nested field don't throw errors
OpenSearchQueryBuilderFactory factory = new OpenSearchQueryBuilderFactory();

OMQueryBuilder nested =
factory.nestedQuery("owners", factory.termQuery("owners.id", "user-abc"));
Query query = ((OpenSearchQueryBuilder) nested).build();
String json = query.toJsonString();

assertTrue(
json.contains("\"ignore_unmapped\":true"),
"must set ignore_unmapped so query works on indexes without this nested field");
}

@Test
void testStaticNestedQueryWorksForMappedFields() {
Query inner =
Query.of(q -> q.term(t -> t.field("owners.id").value(v -> v.stringValue("team-1"))));

Query nested =
org.openmetadata.service.search.opensearch.OpenSearchQueryBuilder.nestedQuery(
"owners", inner);
String json = nested.toJsonString();

assertTrue(json.contains("\"path\":\"owners\""), "must target correct nested path");
assertTrue(json.contains("\"owners.id\""), "must query the nested field");
assertTrue(json.contains("\"team-1\""), "must contain the search value");
}

@Test
void testStaticNestedQueryDoesNotFailForUnmappedFields() {
Query inner =
Query.of(q -> q.term(t -> t.field("owners.id").value(v -> v.stringValue("team-1"))));

Query nested =
org.openmetadata.service.search.opensearch.OpenSearchQueryBuilder.nestedQuery(
"owners", inner);
String json = nested.toJsonString();

assertTrue(
json.contains("\"ignore_unmapped\":true"),
"must set ignore_unmapped so query works on indexes without this nested field");
}
}
Loading