From 89122f6e2b711c1303535c8b03cc7d9e7c64bbc8 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 14:50:25 +0530 Subject: [PATCH 01/22] Fix #27107: soft-deleted users still appear in experts/reviewers across all entities Two bugs caused soft-deleted users to leak into experts/reviewers/owners/followers: 1. EntityRepository.resolveRelationshipEntityReferencesByType hardcoded Include.ALL for the consolidated bulk list path, affecting ALL entities (DataProduct, Domain, GlossaryTerm, Glossary, Table, Schema, Service, Container, etc.). Fixed by switching to getEntityReferencesByIdsRespectingInclude(..., NON_DELETED). 2. EntityResource.getInternal / getByNameInternal built RelationIncludes(null, ...) which defaulted to Include.ALL, causing single-entity GET on DataProduct, Domain, EventSubscription, Query to return deleted users in relation fields. Fixed by defaulting null include to Include.NON_DELETED for all resource endpoints. Also cleaned up dead-code in DataProductRepository.batchFetchExperts and DomainRepository.batchFetchExperts which used getEntityReferenceById that silently overrides NON_DELETED to ALL. Adds regression tests in DataProductResourceIT, DomainResourceIT, GlossaryTermResourceIT covering both single-GET and list-endpoint paths. --- .../it/tests/DataProductResourceIT.java | 76 +++++++++++++++++++ .../it/tests/DomainResourceIT.java | 73 ++++++++++++++++++ .../it/tests/GlossaryTermResourceIT.java | 38 ++++++++++ .../service/jdbi3/DataProductRepository.java | 19 +++-- .../service/jdbi3/DomainRepository.java | 19 +++-- .../service/jdbi3/EntityRepository.java | 3 +- .../service/resources/EntityResource.java | 6 +- 7 files changed, 219 insertions(+), 15 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java index 8e7af3c005a8..a8ca059e9600 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java @@ -33,6 +33,7 @@ import org.openmetadata.schema.api.domains.CreateDomain.DomainType; import org.openmetadata.schema.api.domains.DataProductPortsView; import org.openmetadata.schema.api.services.CreateDatabaseService; +import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.entity.data.Dashboard; import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.data.Topic; @@ -41,6 +42,7 @@ import org.openmetadata.schema.entity.services.DashboardService; import org.openmetadata.schema.entity.services.DatabaseService; import org.openmetadata.schema.entity.services.MessagingService; +import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.entity.type.Style; import org.openmetadata.schema.services.connections.database.MysqlConnection; import org.openmetadata.schema.services.connections.database.common.basicAuth; @@ -2873,4 +2875,78 @@ void test_deletingAssetRemovesItFromPorts(TestNamespace ns) throws Exception { ResultList> outputPorts = getOutputPorts(dataProduct.getId(), 10, 0); assertEquals(0, outputPorts.getPaging().getTotal()); } + + @Test + void softDeletedExpert_notReturnedInSingleGet(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + Domain domain = getOrCreateDomain(ns); + + String userName = ns.prefix("expert_user"); + User expert = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withDescription("Expert user for soft-delete test")); + + CreateDataProduct create = + new CreateDataProduct() + .withName(ns.prefix("dp_softdel_expert")) + .withDescription("DataProduct for soft-delete expert test") + .withDomains(List.of(domain.getFullyQualifiedName())) + .withExperts(List.of(expert.getFullyQualifiedName())); + DataProduct dp = createEntity(create); + + client.users().delete(expert.getId().toString()); + + DataProduct byId = client.dataProducts().get(dp.getId().toString(), "experts"); + assertTrue( + byId.getExperts() == null || byId.getExperts().isEmpty(), + "Soft-deleted expert must not appear in single GET by ID"); + + DataProduct byName = client.dataProducts().getByName(dp.getFullyQualifiedName(), "experts"); + assertTrue( + byName.getExperts() == null || byName.getExperts().isEmpty(), + "Soft-deleted expert must not appear in single GET by name"); + } + + @Test + void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + Domain domain = getOrCreateDomain(ns); + + String userName = ns.prefix("expert_list_user"); + User expert = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withDescription("Expert user for bulk soft-delete test")); + + CreateDataProduct create = + new CreateDataProduct() + .withName(ns.prefix("dp_softdel_expert_list")) + .withDescription("DataProduct for soft-delete expert list test") + .withDomains(List.of(domain.getFullyQualifiedName())) + .withExperts(List.of(expert.getFullyQualifiedName())); + DataProduct dp = createEntity(create); + + client.users().delete(expert.getId().toString()); + + ListParams params = + new ListParams().setFields("experts").withDomain(domain.getFullyQualifiedName()); + ListResponse list = client.dataProducts().list(params); + DataProduct listed = + list.getData().stream() + .filter(p -> p.getId().equals(dp.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("DataProduct not found in list")); + assertTrue( + listed.getExperts() == null || listed.getExperts().isEmpty(), + "Soft-deleted expert must not appear in list endpoint"); + } } diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java index c36b0bcbd661..760b6cb4e967 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java @@ -31,7 +31,9 @@ import org.openmetadata.it.util.TestNamespace; import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.api.domains.CreateDomain.DomainType; +import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.entity.domains.Domain; +import org.openmetadata.schema.entity.teams.User; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.EntityReference; import org.openmetadata.sdk.client.OpenMetadataClient; @@ -1153,4 +1155,75 @@ void test_renameDomainDoesNotAffectSimilarPrefixDomains(TestNamespace ns) throws // Verify old child FQN no longer works assertThrows(Exception.class, () -> getEntityByName(oldChildFqn)); } + + @Test + void softDeletedExpert_notReturnedInSingleGet(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + + String userName = ns.prefix("domain_expert"); + User expert = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withDescription("Expert user for domain soft-delete test")); + + CreateDomain create = + new CreateDomain() + .withName(ns.prefix("domain_softdel")) + .withDomainType(DomainType.AGGREGATE) + .withExperts(List.of(expert.getFullyQualifiedName())) + .withDescription("Domain for soft-delete expert test"); + Domain domain = createEntity(create); + + client.users().delete(expert.getId().toString()); + + Domain byId = client.domains().get(domain.getId().toString(), "experts"); + assertTrue( + byId.getExperts() == null || byId.getExperts().isEmpty(), + "Soft-deleted expert must not appear in single GET by ID"); + + Domain byName = client.domains().getByName(domain.getFullyQualifiedName(), "experts"); + assertTrue( + byName.getExperts() == null || byName.getExperts().isEmpty(), + "Soft-deleted expert must not appear in single GET by name"); + } + + @Test + void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + + String userName = ns.prefix("domain_expert_list"); + User expert = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withDescription("Expert user for domain list soft-delete test")); + + CreateDomain create = + new CreateDomain() + .withName(ns.prefix("domain_softdel_list")) + .withDomainType(DomainType.AGGREGATE) + .withExperts(List.of(expert.getFullyQualifiedName())) + .withDescription("Domain for soft-delete expert list test"); + Domain domain = createEntity(create); + + client.users().delete(expert.getId().toString()); + + ListParams params = new ListParams().setFields("experts").withLimit(100); + ListResponse list = listEntities(params); + Domain listed = + list.getData().stream() + .filter(d -> d.getId().equals(domain.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("Domain not found in list")); + assertTrue( + listed.getExperts() == null || listed.getExperts().isEmpty(), + "Soft-deleted expert must not appear in list endpoint"); + } } diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java index 36968e8c6d7f..991c0df1770a 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java @@ -28,6 +28,7 @@ import org.openmetadata.schema.api.data.CreateTable; import org.openmetadata.schema.api.data.TermReference; import org.openmetadata.schema.api.feed.CreateThread; +import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.entity.data.DatabaseSchema; import org.openmetadata.schema.entity.data.Glossary; import org.openmetadata.schema.entity.data.GlossaryTerm; @@ -3147,4 +3148,41 @@ private String getTermAssetsByName(OpenMetadataClient client, String fqn) { null, optionsBuilder.build()); } + + @Test + void softDeletedReviewer_notReturnedInListEndpoint(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + Glossary glossary = getOrCreateGlossary(ns); + + String userName = ns.prefix("reviewer_list"); + User reviewer = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withDescription("Reviewer user for glossary soft-delete list test")); + + CreateGlossaryTerm create = + new CreateGlossaryTerm() + .withName(ns.prefix("term_softdel_reviewer")) + .withGlossary(glossary.getFullyQualifiedName()) + .withDescription("Term for soft-delete reviewer list test") + .withReviewers(List.of(reviewer.getEntityReference())); + GlossaryTerm term = createEntity(create); + + client.users().delete(reviewer.getId().toString()); + + ListParams params = new ListParams().setFields("reviewers").withLimit(100); + ListResponse list = listEntities(params); + GlossaryTerm listed = + list.getData().stream() + .filter(t -> t.getId().equals(term.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("GlossaryTerm not found in list")); + assertTrue( + listed.getReviewers() == null || listed.getReviewers().isEmpty(), + "Soft-deleted reviewer must not appear in list endpoint"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index b5f815d18c45..6a1e4fc96d28 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -38,6 +38,7 @@ import java.util.Map; import java.util.Set; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -967,24 +968,30 @@ private Map> batchFetchExperts(List dat return expertsMap; } - // Initialize empty lists for all data products for (DataProduct dataProduct : dataProducts) { expertsMap.put(dataProduct.getId(), new ArrayList<>()); } - // Single batch query to get all expert relationships List records = daoCollection .relationshipDAO() .findToBatch( entityListToStrings(dataProducts), Relationship.EXPERT.ordinal(), Entity.USER); - // Group experts by data product ID + List expertIds = + records.stream().map(r -> UUID.fromString(r.getToId())).distinct().toList(); + Map expertRefsById = + Entity.getEntityReferencesByIdsRespectingInclude( + Entity.USER, expertIds, Include.NON_DELETED) + .stream() + .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + for (CollectionDAO.EntityRelationshipObject record : records) { UUID dataProductId = UUID.fromString(record.getFromId()); - EntityReference expertRef = - Entity.getEntityReferenceById( - Entity.USER, UUID.fromString(record.getToId()), NON_DELETED); + EntityReference expertRef = expertRefsById.get(UUID.fromString(record.getToId())); + if (expertRef == null) { + continue; + } expertsMap.get(dataProductId).add(expertRef); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 615e0ebc717e..6b991ae965ee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -27,6 +27,7 @@ import java.util.List; import java.util.Map; import java.util.UUID; +import java.util.function.Function; import java.util.stream.Collectors; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.sqlobject.transaction.Transaction; @@ -580,22 +581,28 @@ private Map> batchFetchExperts(List domains) return expertsMap; } - // Initialize empty lists for all domains domains.forEach(domain -> expertsMap.put(domain.getId(), new ArrayList<>())); - // Single batch query to get all expert relationships var records = daoCollection .relationshipDAO() .findToBatch(entityListToStrings(domains), Relationship.EXPERT.ordinal(), Entity.USER); - // Group experts by domain ID + List expertIds = + records.stream().map(r -> UUID.fromString(r.getToId())).distinct().toList(); + Map expertRefsById = + Entity.getEntityReferencesByIdsRespectingInclude( + Entity.USER, expertIds, Include.NON_DELETED) + .stream() + .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + records.forEach( record -> { var domainId = UUID.fromString(record.getFromId()); - var expertRef = - getEntityReferenceById(Entity.USER, UUID.fromString(record.getToId()), NON_DELETED); - expertsMap.get(domainId).add(expertRef); + var expertRef = expertRefsById.get(UUID.fromString(record.getToId())); + if (expertRef != null) { + expertsMap.get(domainId).add(expertRef); + } }); return expertsMap; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index becfa4fccc1a..bcec2296e3b3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -8704,7 +8704,8 @@ private Map> resolveRelationshipEntityReferen Map> refsByType = new HashMap<>(); for (Entry> entry : idsByType.entrySet()) { List refs = - Entity.getEntityReferencesByIds(entry.getKey(), new ArrayList<>(entry.getValue()), ALL); + Entity.getEntityReferencesByIdsRespectingInclude( + entry.getKey(), new ArrayList<>(entry.getValue()), Include.NON_DELETED); refsByType.put( entry.getKey(), refs.stream() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java index bee61836e6c8..331d320d55e2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java @@ -285,7 +285,8 @@ public T getInternal( String includeRelations) { Fields fields = getFields(fieldsParam); OperationContext operationContext = new OperationContext(entityType, getViewOperations(fields)); - RelationIncludes relationIncludes = new RelationIncludes(include, includeRelations); + Include resolvedInclude = include != null ? include : Include.NON_DELETED; + RelationIncludes relationIncludes = new RelationIncludes(resolvedInclude, includeRelations); return getInternal( uriInfo, securityContext, @@ -388,7 +389,8 @@ public T getByNameInternal( String includeRelations) { Fields fields = getFields(fieldsParam); OperationContext operationContext = new OperationContext(entityType, getViewOperations(fields)); - RelationIncludes relationIncludes = new RelationIncludes(include, includeRelations); + Include resolvedInclude = include != null ? include : Include.NON_DELETED; + RelationIncludes relationIncludes = new RelationIncludes(resolvedInclude, includeRelations); return getByNameInternal( uriInfo, securityContext, From f56057c59d4d9c639d21b7db203325e149f8620d Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 15:44:49 +0530 Subject: [PATCH 02/22] Thread Include through bulk relationship resolution instead of hardcoding NON_DELETED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Add Include parameter to resolveRelationshipEntityReferencesByType, fetchAndSetRelationshipFieldsInBulk, and fetchAndSetFields (new 3-arg overload) - Add setFieldsInBulk(fields, entities, Include) overload in base class; existing 2-arg delegates to NON_DELETED - The Include is no longer hardcoded — callers with request context (e.g. include=all) can pass it through via the 3-arg setFieldsInBulk or fetchAndSetFields Fixes: open-metadata/OpenMetadata#27107 --- .../service/jdbi3/EntityRepository.java | 24 +++++++++++++------ 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index bcec2296e3b3..641f242591e9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1879,11 +1879,15 @@ public final List listAllForCSV(Fields fields, String parentFqn) { * Example implementation can be found in {@link GlossaryTermRepository#setFieldsInBulk}. */ public void setFieldsInBulk(Fields fields, List entities) { + setFieldsInBulk(fields, entities, Include.NON_DELETED); + } + + public void setFieldsInBulk(Fields fields, List entities, Include include) { if (entities == null || entities.isEmpty()) { return; } try (var ignored = phase("fetchFields")) { - fetchAndSetFields(entities, fields); + fetchAndSetFields(entities, fields, include); } try (var ignored = phase("setInheritedFields")) { setInheritedFields(entities, fields); @@ -8421,7 +8425,12 @@ public static void validateColumn(Table table, String columnName, Boolean caseSe } protected void fetchAndSetFields(List entities, Fields fields) { - Set relationshipFieldsHandled = fetchAndSetRelationshipFieldsInBulk(entities, fields); + fetchAndSetFields(entities, fields, Include.NON_DELETED); + } + + protected void fetchAndSetFields(List entities, Fields fields, Include include) { + Set relationshipFieldsHandled = + fetchAndSetRelationshipFieldsInBulk(entities, fields, include); for (Entry, Fields>> entry : fieldFetchers.entrySet()) { if (relationshipFieldsHandled.contains(entry.getKey())) { continue; @@ -8430,7 +8439,8 @@ protected void fetchAndSetFields(List entities, Fields fields) { } } - private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields fields) { + private Set fetchAndSetRelationshipFieldsInBulk( + List entities, Fields fields, Include include) { if (nullOrEmpty(entities) || fields == null) { return Collections.emptySet(); } @@ -8497,9 +8507,9 @@ private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields .findToBatchWithRelations(entityIds, entityType, outgoingRelations, ALL); Map> incomingRefsByType = - resolveRelationshipEntityReferencesByType(incomingRecords, true); + resolveRelationshipEntityReferencesByType(incomingRecords, true, include); Map> outgoingRefsByType = - resolveRelationshipEntityReferencesByType(outgoingRecords, false); + resolveRelationshipEntityReferencesByType(outgoingRecords, false, include); Map> ownersByEntity = loadOwners ? new HashMap<>() : null; Map> followersByEntity = loadFollowers ? new HashMap<>() : null; @@ -8680,7 +8690,7 @@ private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields } private Map> resolveRelationshipEntityReferencesByType( - List records, boolean fromSide) { + List records, boolean fromSide, Include include) { if (records == null || records.isEmpty()) { return Collections.emptyMap(); } @@ -8705,7 +8715,7 @@ private Map> resolveRelationshipEntityReferen for (Entry> entry : idsByType.entrySet()) { List refs = Entity.getEntityReferencesByIdsRespectingInclude( - entry.getKey(), new ArrayList<>(entry.getValue()), Include.NON_DELETED); + entry.getKey(), new ArrayList<>(entry.getValue()), include); refsByType.put( entry.getKey(), refs.stream() From 3264fb34f7ea3049bdb61faaa78759c3ad515c33 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 17:02:44 +0530 Subject: [PATCH 03/22] Fix inherited experts/owners including soft-deleted users from parent Domain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit DataProductRepository.setInheritedFields and DomainRepository.setInheritedFields both fetched parent Domain entities with Include.ALL, causing the domain's experts and owners fields to be resolved with ALL include — returning soft-deleted users. Changed to NON_DELETED so inherited fields only carry active users. Fixes: open-metadata/OpenMetadata#27107 --- .../org/openmetadata/service/jdbi3/DataProductRepository.java | 2 +- .../java/org/openmetadata/service/jdbi3/DomainRepository.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 6a1e4fc96d28..7c7c7c83e91a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -209,7 +209,7 @@ public void setInheritedFields(DataProduct dataProduct, Fields fields) { List experts = new ArrayList<>(); for (EntityReference domainRef : domains) { - Domain domain = Entity.getEntity(DOMAIN, domainRef.getId(), "owners,experts", ALL); + Domain domain = Entity.getEntity(DOMAIN, domainRef.getId(), "owners,experts", NON_DELETED); owners = mergedInheritedEntityRefs(owners, domain.getOwners()); experts = mergedInheritedEntityRefs(experts, domain.getExperts()); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 6b991ae965ee..1a6c64d960fd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -186,7 +186,7 @@ public void setInheritedFields(Domain domain, Fields fields) { // domain EntityReference parentRef = domain.getParent() != null ? domain.getParent() : getParent(domain); if (parentRef != null) { - Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", ALL); + Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", NON_DELETED); inheritOwners(domain, fields, parent); inheritExperts(domain, fields, parent); } From 841775b0722afb9a8f97510add4222c35f5a3ec4 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 17:12:21 +0530 Subject: [PATCH 04/22] Fix soft-deleted users leaking into owners/experts/reviewers across all entities MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five additional bug sites using Include.ALL when resolving user relationships: - EntityRepository.batchFetchOwners: getEntityReferencesByIds → getEntityReferencesByIdsRespectingInclude(NON_DELETED) - EntityRepository.batchFetchReviewers: same fix - EntityRepository.batchFetchExperts (base class): same fix - TagRepository.setInheritedFields: fetch Classification with NON_DELETED (was ALL), preventing soft-deleted owners/reviewers from being inherited by Tags - WorksheetRepository.setInheritedFields: fetch Spreadsheet with NON_DELETED (was ALL), preventing soft-deleted owners from being inherited by Worksheets Fixes: open-metadata/OpenMetadata#27107 --- .../org/openmetadata/service/jdbi3/EntityRepository.java | 8 +++++--- .../org/openmetadata/service/jdbi3/TagRepository.java | 5 ++++- .../openmetadata/service/jdbi3/WorksheetRepository.java | 5 ++++- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 641f242591e9..73b005cb5bee 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -8928,7 +8928,8 @@ private Map> batchFetchOwners(List entities) { ownerIdsByType.forEach( (entityType, ownerIds) -> { var ownerRefs = - Entity.getEntityReferencesByIds(entityType, new ArrayList<>(ownerIds), ALL); + Entity.getEntityReferencesByIdsRespectingInclude( + entityType, new ArrayList<>(ownerIds), NON_DELETED); var refMap = ownerRefs.stream() .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); @@ -9213,7 +9214,8 @@ private Map> batchFetchReviewers(List entities) { reviewerIdsByType.forEach( (entityType, reviewerIds) -> { var reviewerRefs = - Entity.getEntityReferencesByIds(entityType, new ArrayList<>(reviewerIds), ALL); + Entity.getEntityReferencesByIdsRespectingInclude( + entityType, new ArrayList<>(reviewerIds), NON_DELETED); var refMap = reviewerRefs.stream() .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); @@ -9297,7 +9299,7 @@ private Map> batchFetchExperts(List entities) { // Batch fetch all expert references Map expertRefs = - Entity.getEntityReferencesByIds(USER, expertIds, ALL).stream() + Entity.getEntityReferencesByIdsRespectingInclude(USER, expertIds, NON_DELETED).stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); // Group experts by entity (reuse cached UUIDs) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java index 847e8e22580a..be8595b6a2a0 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java @@ -238,7 +238,10 @@ public void setInheritedFields(Tag tag, Fields fields) { try { Classification parent = Entity.getEntity( - CLASSIFICATION, tag.getClassification().getId(), "owners,domains,reviewers", ALL); + CLASSIFICATION, + tag.getClassification().getId(), + "owners,domains,reviewers", + NON_DELETED); if (parent.getDisabled() != null && parent.getDisabled()) { tag.setDisabled(true); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java index edbd72a01b11..6b6f71e7776f 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java @@ -222,7 +222,10 @@ public void setInheritedFields(Worksheet worksheet, EntityUtil.Fields fields) { if (worksheet.getSpreadsheet() != null) { Spreadsheet spreadsheet = Entity.getEntity( - SPREADSHEET, worksheet.getSpreadsheet().getId(), "owners,domains", Include.ALL); + SPREADSHEET, + worksheet.getSpreadsheet().getId(), + "owners,domains", + Include.NON_DELETED); inheritOwners(worksheet, fields, spreadsheet); inheritDomains(worksheet, fields, spreadsheet); } From d4743a1407d734d0c3a407e923a6971808331f94 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 17:58:24 +0530 Subject: [PATCH 05/22] Fix test user email validation in soft-delete integration tests ns.prefix() generates ~110-char strings; after stripping underscores the local part exceeds RFC 5321's 64-char limit and the server's maxLength:127 constraint. Switch to ns.shortPrefix() (~27 chars total) which is already alphanumeric+underscore and requires no sanitisation. --- .../org/openmetadata/it/tests/DataProductResourceIT.java | 8 ++++---- .../java/org/openmetadata/it/tests/DomainResourceIT.java | 8 ++++---- .../org/openmetadata/it/tests/GlossaryTermResourceIT.java | 4 ++-- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java index a8ca059e9600..bc8219a87c1a 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java @@ -2881,14 +2881,14 @@ void softDeletedExpert_notReturnedInSingleGet(TestNamespace ns) { OpenMetadataClient client = SdkClients.adminClient(); Domain domain = getOrCreateDomain(ns); - String userName = ns.prefix("expert_user"); + String userName = ns.shortPrefix("expert_user"); User expert = client .users() .create( new CreateUser() .withName(userName) - .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withEmail(userName + "@test.openmetadata.org") .withDescription("Expert user for soft-delete test")); CreateDataProduct create = @@ -2917,14 +2917,14 @@ void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { OpenMetadataClient client = SdkClients.adminClient(); Domain domain = getOrCreateDomain(ns); - String userName = ns.prefix("expert_list_user"); + String userName = ns.shortPrefix("expert_list_user"); User expert = client .users() .create( new CreateUser() .withName(userName) - .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withEmail(userName + "@test.openmetadata.org") .withDescription("Expert user for bulk soft-delete test")); CreateDataProduct create = diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java index 760b6cb4e967..96055803d9f6 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java @@ -1160,14 +1160,14 @@ void test_renameDomainDoesNotAffectSimilarPrefixDomains(TestNamespace ns) throws void softDeletedExpert_notReturnedInSingleGet(TestNamespace ns) { OpenMetadataClient client = SdkClients.adminClient(); - String userName = ns.prefix("domain_expert"); + String userName = ns.shortPrefix("domain_expert"); User expert = client .users() .create( new CreateUser() .withName(userName) - .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withEmail(userName + "@test.openmetadata.org") .withDescription("Expert user for domain soft-delete test")); CreateDomain create = @@ -1195,14 +1195,14 @@ void softDeletedExpert_notReturnedInSingleGet(TestNamespace ns) { void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { OpenMetadataClient client = SdkClients.adminClient(); - String userName = ns.prefix("domain_expert_list"); + String userName = ns.shortPrefix("domain_expert_list"); User expert = client .users() .create( new CreateUser() .withName(userName) - .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withEmail(userName + "@test.openmetadata.org") .withDescription("Expert user for domain list soft-delete test")); CreateDomain create = diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java index 991c0df1770a..c48d1e78d5a3 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java @@ -3154,14 +3154,14 @@ void softDeletedReviewer_notReturnedInListEndpoint(TestNamespace ns) { OpenMetadataClient client = SdkClients.adminClient(); Glossary glossary = getOrCreateGlossary(ns); - String userName = ns.prefix("reviewer_list"); + String userName = ns.shortPrefix("reviewer_list"); User reviewer = client .users() .create( new CreateUser() .withName(userName) - .withEmail(userName.replaceAll("[^a-zA-Z0-9]", "") + "@test.openmetadata.org") + .withEmail(userName + "@test.openmetadata.org") .withDescription("Reviewer user for glossary soft-delete list test")); CreateGlossaryTerm create = From 05b41cb665dccb9508d61f94e16c3a1889628b07 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 18:07:23 +0530 Subject: [PATCH 06/22] Fix GlossaryTerm list test to filter by glossary ID Without a glossary filter the list call returns the first 100 terms across all glossaries; in a populated CI env the test term lands outside that window and the search fails with 'GlossaryTerm not found in list'. Add .addFilter("glossary", glossary.getId()) so the query is scoped to only our test glossary. --- .../org/openmetadata/it/tests/GlossaryTermResourceIT.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java index c48d1e78d5a3..f5c7e04e9fb4 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/GlossaryTermResourceIT.java @@ -3174,7 +3174,11 @@ void softDeletedReviewer_notReturnedInListEndpoint(TestNamespace ns) { client.users().delete(reviewer.getId().toString()); - ListParams params = new ListParams().setFields("reviewers").withLimit(100); + ListParams params = + new ListParams() + .setFields("reviewers") + .withLimit(100) + .addFilter("glossary", glossary.getId().toString()); ListResponse list = listEntities(params); GlossaryTerm listed = list.getData().stream() From 1a1888a810aac0635bf6ff90e5f8c8e63fec5272 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 21:08:26 +0530 Subject: [PATCH 07/22] Fix fromId/toId swap in EntityRepository.batchFetchExperts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit findToBatch(entityIds, EXPERT, USER) returns rows where fromId=entity and toId=user (matching the write path in bulkInsertToRelationship). The original code collected fromId as expert user IDs and toId as entity IDs — both backwards. Method is preempted by the bulk path in practice, but the logic should be correct. --- .../openmetadata/service/jdbi3/EntityRepository.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 73b005cb5bee..ac47d7bf6806 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -9290,23 +9290,23 @@ private Map> batchFetchExperts(List entities) { // Cache UUID conversions to avoid repeated parsing Map uuidCache = new HashMap<>(); - // Collect all unique expert user IDs (with .distinct() to avoid duplicate fetches) + // findToBatch returns fromId=entity, toId=user — collect user IDs from toId List expertIds = records.stream() - .map(record -> uuidCache.computeIfAbsent(record.getFromId(), UUID::fromString)) + .map(record -> uuidCache.computeIfAbsent(record.getToId(), UUID::fromString)) .distinct() .collect(Collectors.toList()); - // Batch fetch all expert references + // Batch fetch all expert references, filtering out soft-deleted users Map expertRefs = Entity.getEntityReferencesByIdsRespectingInclude(USER, expertIds, NON_DELETED).stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); - // Group experts by entity (reuse cached UUIDs) + // Group experts by entity records.forEach( record -> { - UUID entityId = uuidCache.computeIfAbsent(record.getToId(), UUID::fromString); - UUID expertId = uuidCache.get(record.getFromId()); // Already cached above + UUID entityId = uuidCache.computeIfAbsent(record.getFromId(), UUID::fromString); + UUID expertId = uuidCache.get(record.getToId()); // Already cached above EntityReference expertRef = expertRefs.get(expertId); if (expertRef != null) { expertsMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(expertRef); From 0c450067166410260f454dc0da6b035ef6b02e46 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 21:14:59 +0530 Subject: [PATCH 08/22] Add merge function to Collectors.toMap in batchFetchExperts Prevents potential IllegalStateException if getEntityReferencesByIdsRespectingInclude returns duplicate references for the same ID due to data inconsistency. Consistent with the (a, b) -> a merge function used elsewhere. --- .../org/openmetadata/service/jdbi3/DataProductRepository.java | 2 +- .../java/org/openmetadata/service/jdbi3/DomainRepository.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 7c7c7c83e91a..804936942535 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -984,7 +984,7 @@ private Map> batchFetchExperts(List dat Entity.getEntityReferencesByIdsRespectingInclude( Entity.USER, expertIds, Include.NON_DELETED) .stream() - .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); for (CollectionDAO.EntityRelationshipObject record : records) { UUID dataProductId = UUID.fromString(record.getFromId()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 1a6c64d960fd..3be5baa7d240 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -594,7 +594,7 @@ private Map> batchFetchExperts(List domains) Entity.getEntityReferencesByIdsRespectingInclude( Entity.USER, expertIds, Include.NON_DELETED) .stream() - .collect(Collectors.toMap(EntityReference::getId, Function.identity())); + .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); records.forEach( record -> { From 639c950c2d9c465821b63928c7a72977f129ad30 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 21:57:49 +0530 Subject: [PATCH 09/22] Handle soft-deleted parent in setInheritedFields for DataProduct and Domain Entity.getEntity(..., NON_DELETED) throws EntityNotFoundException when the parent domain/domain is soft-deleted while the relationship still exists. Catch and skip inheritance for that parent rather than propagating the exception and breaking the GET/list response. --- .../service/jdbi3/DataProductRepository.java | 14 +++++++++++--- .../service/jdbi3/DomainRepository.java | 14 +++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 804936942535..86f621537c91 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -209,9 +209,17 @@ public void setInheritedFields(DataProduct dataProduct, Fields fields) { List experts = new ArrayList<>(); for (EntityReference domainRef : domains) { - Domain domain = Entity.getEntity(DOMAIN, domainRef.getId(), "owners,experts", NON_DELETED); - owners = mergedInheritedEntityRefs(owners, domain.getOwners()); - experts = mergedInheritedEntityRefs(experts, domain.getExperts()); + try { + Domain domain = + Entity.getEntity(DOMAIN, domainRef.getId(), "owners,experts", NON_DELETED); + owners = mergedInheritedEntityRefs(owners, domain.getOwners()); + experts = mergedInheritedEntityRefs(experts, domain.getExperts()); + } catch (EntityNotFoundException ex) { + LOG.debug( + "Skipping inherited fields from soft-deleted or missing domain {} for data product {}", + domainRef.getId(), + dataProduct.getId()); + } } // inherit only if applicable and empty if (inheritOwners) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 3be5baa7d240..2033b73ca488 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -48,6 +48,7 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.schema.utils.ResultList; import org.openmetadata.service.Entity; +import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.resources.domains.DomainResource; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.DefaultInheritedFieldEntitySearch; @@ -186,9 +187,16 @@ public void setInheritedFields(Domain domain, Fields fields) { // domain EntityReference parentRef = domain.getParent() != null ? domain.getParent() : getParent(domain); if (parentRef != null) { - Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", NON_DELETED); - inheritOwners(domain, fields, parent); - inheritExperts(domain, fields, parent); + try { + Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", NON_DELETED); + inheritOwners(domain, fields, parent); + inheritExperts(domain, fields, parent); + } catch (EntityNotFoundException ex) { + LOG.debug( + "Skipping inherited fields from soft-deleted or missing parent domain {} for domain {}", + parentRef.getId(), + domain.getId()); + } } } From 97f328ac9b736feb9ba4ebef578bf1c8c9eb1fd8 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Tue, 7 Apr 2026 22:16:14 +0530 Subject: [PATCH 10/22] Remove dead setFieldsInBulk(Include) overload and collapse Include from private chain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 3-arg setFieldsInBulk(Fields, List, Include) overload had no callers passing a non-default Include — the intent of this PR is to always filter soft-deleted users from relationship fields, not to respect the caller's include. Remove the overload and collapse the Include parameter out of fetchAndSetFields and fetchAndSetRelationshipFieldsInBulk/ resolveRelationshipEntityReferencesByType, hardcoding NON_DELETED where it matters. --- .../service/jdbi3/EntityRepository.java | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index ac47d7bf6806..7ba71d1e240b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1879,15 +1879,11 @@ public final List listAllForCSV(Fields fields, String parentFqn) { * Example implementation can be found in {@link GlossaryTermRepository#setFieldsInBulk}. */ public void setFieldsInBulk(Fields fields, List entities) { - setFieldsInBulk(fields, entities, Include.NON_DELETED); - } - - public void setFieldsInBulk(Fields fields, List entities, Include include) { if (entities == null || entities.isEmpty()) { return; } try (var ignored = phase("fetchFields")) { - fetchAndSetFields(entities, fields, include); + fetchAndSetFields(entities, fields); } try (var ignored = phase("setInheritedFields")) { setInheritedFields(entities, fields); @@ -8425,12 +8421,7 @@ public static void validateColumn(Table table, String columnName, Boolean caseSe } protected void fetchAndSetFields(List entities, Fields fields) { - fetchAndSetFields(entities, fields, Include.NON_DELETED); - } - - protected void fetchAndSetFields(List entities, Fields fields, Include include) { - Set relationshipFieldsHandled = - fetchAndSetRelationshipFieldsInBulk(entities, fields, include); + Set relationshipFieldsHandled = fetchAndSetRelationshipFieldsInBulk(entities, fields); for (Entry, Fields>> entry : fieldFetchers.entrySet()) { if (relationshipFieldsHandled.contains(entry.getKey())) { continue; @@ -8439,8 +8430,7 @@ protected void fetchAndSetFields(List entities, Fields fields, Include includ } } - private Set fetchAndSetRelationshipFieldsInBulk( - List entities, Fields fields, Include include) { + private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields fields) { if (nullOrEmpty(entities) || fields == null) { return Collections.emptySet(); } @@ -8507,9 +8497,9 @@ private Set fetchAndSetRelationshipFieldsInBulk( .findToBatchWithRelations(entityIds, entityType, outgoingRelations, ALL); Map> incomingRefsByType = - resolveRelationshipEntityReferencesByType(incomingRecords, true, include); + resolveRelationshipEntityReferencesByType(incomingRecords, true); Map> outgoingRefsByType = - resolveRelationshipEntityReferencesByType(outgoingRecords, false, include); + resolveRelationshipEntityReferencesByType(outgoingRecords, false); Map> ownersByEntity = loadOwners ? new HashMap<>() : null; Map> followersByEntity = loadFollowers ? new HashMap<>() : null; @@ -8690,7 +8680,7 @@ private Set fetchAndSetRelationshipFieldsInBulk( } private Map> resolveRelationshipEntityReferencesByType( - List records, boolean fromSide, Include include) { + List records, boolean fromSide) { if (records == null || records.isEmpty()) { return Collections.emptyMap(); } @@ -8715,7 +8705,7 @@ private Map> resolveRelationshipEntityReferen for (Entry> entry : idsByType.entrySet()) { List refs = Entity.getEntityReferencesByIdsRespectingInclude( - entry.getKey(), new ArrayList<>(entry.getValue()), include); + entry.getKey(), new ArrayList<>(entry.getValue()), Include.NON_DELETED); refsByType.put( entry.getKey(), refs.stream() From d9aa379505ea9cd3bd8375f7c9b77ecffc4b852a Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 00:02:44 +0530 Subject: [PATCH 11/22] Paginate domain list to find test domain regardless of total count --- .../it/tests/DomainResourceIT.java | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java index 96055803d9f6..babcfc25d500 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java @@ -1215,13 +1215,20 @@ void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { client.users().delete(expert.getId().toString()); + Domain listed = null; ListParams params = new ListParams().setFields("experts").withLimit(100); - ListResponse list = listEntities(params); - Domain listed = - list.getData().stream() - .filter(d -> d.getId().equals(domain.getId())) - .findFirst() - .orElseThrow(() -> new AssertionError("Domain not found in list")); + while (listed == null) { + ListResponse page = listEntities(params); + listed = + page.getData().stream() + .filter(d -> d.getId().equals(domain.getId())) + .findFirst() + .orElse(null); + String after = page.getPaging() != null ? page.getPaging().getAfter() : null; + if (listed != null || after == null) break; + params = new ListParams().setFields("experts").withLimit(100).setAfter(after); + } + assertNotNull(listed, "Domain not found in list"); assertTrue( listed.getExperts() == null || listed.getExperts().isEmpty(), "Soft-deleted expert must not appear in list endpoint"); From 37a014a601e5e3236c6d175efd49a4bdc54c098e Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 00:31:45 +0530 Subject: [PATCH 12/22] =?UTF-8?q?Revert=20WorksheetRepository=20Include=20?= =?UTF-8?q?change=20=E2=80=94=20out=20of=20scope=20for=20#27107?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../org/openmetadata/service/jdbi3/WorksheetRepository.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java index 6b6f71e7776f..edbd72a01b11 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/WorksheetRepository.java @@ -222,10 +222,7 @@ public void setInheritedFields(Worksheet worksheet, EntityUtil.Fields fields) { if (worksheet.getSpreadsheet() != null) { Spreadsheet spreadsheet = Entity.getEntity( - SPREADSHEET, - worksheet.getSpreadsheet().getId(), - "owners,domains", - Include.NON_DELETED); + SPREADSHEET, worksheet.getSpreadsheet().getId(), "owners,domains", Include.ALL); inheritOwners(worksheet, fields, spreadsheet); inheritDomains(worksheet, fields, spreadsheet); } From 4af35c3b8881ffc33ae9856b9d1f47aebff48215 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 19:23:23 +0530 Subject: [PATCH 13/22] Address PR review: add limit to DataProduct list test; align TagRepository bulk path to NON_DELETED --- .../org/openmetadata/it/tests/DataProductResourceIT.java | 5 ++++- .../java/org/openmetadata/service/jdbi3/TagRepository.java | 3 +-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java index bc8219a87c1a..d019280a93e6 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java @@ -2938,7 +2938,10 @@ void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { client.users().delete(expert.getId().toString()); ListParams params = - new ListParams().setFields("experts").withDomain(domain.getFullyQualifiedName()); + new ListParams() + .setFields("experts") + .withDomain(domain.getFullyQualifiedName()) + .withLimit(100); ListResponse list = client.dataProducts().list(params); DataProduct listed = list.getData().stream() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java index be8595b6a2a0..971a7a646fcf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java @@ -15,7 +15,6 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; -import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.CLASSIFICATION; import static org.openmetadata.service.Entity.FIELD_CERTIFICATION; @@ -280,7 +279,7 @@ public void setInheritedFields(List tags, Fields fields) { List classifications = classificationRepository .getDao() - .findEntitiesByIds(new ArrayList<>(classificationIds), ALL); + .findEntitiesByIds(new ArrayList<>(classificationIds), NON_DELETED); classificationRepository.setFieldsInBulk( new Fields(Set.of("owners", "domains", "reviewers")), classifications); From b3d74827ea209eeedbe3b2ae138beefbe1f2c3a8 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 19:26:41 +0530 Subject: [PATCH 14/22] =?UTF-8?q?Revert=20TagRepository=20Include=20change?= =?UTF-8?q?s=20=E2=80=94=20out=20of=20scope=20for=20#27107?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/org/openmetadata/service/jdbi3/TagRepository.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java index 971a7a646fcf..b016a99f2055 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java @@ -237,10 +237,7 @@ public void setInheritedFields(Tag tag, Fields fields) { try { Classification parent = Entity.getEntity( - CLASSIFICATION, - tag.getClassification().getId(), - "owners,domains,reviewers", - NON_DELETED); + CLASSIFICATION, tag.getClassification().getId(), "owners,domains,reviewers", ALL); if (parent.getDisabled() != null && parent.getDisabled()) { tag.setDisabled(true); } @@ -279,7 +276,7 @@ public void setInheritedFields(List tags, Fields fields) { List classifications = classificationRepository .getDao() - .findEntitiesByIds(new ArrayList<>(classificationIds), NON_DELETED); + .findEntitiesByIds(new ArrayList<>(classificationIds), ALL); classificationRepository.setFieldsInBulk( new Fields(Set.of("owners", "domains", "reviewers")), classifications); From d5446b1f8fc6b5114e360f5db6dfcd059780fc4d Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 21:08:03 +0530 Subject: [PATCH 15/22] Remove unnecessary changes and compilation error --- .../service/jdbi3/DataProductRepository.java | 14 +++----------- .../service/jdbi3/DomainRepository.java | 14 +++----------- .../openmetadata/service/jdbi3/TagRepository.java | 1 + 3 files changed, 7 insertions(+), 22 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 86f621537c91..9ea88de01fcd 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -209,17 +209,9 @@ public void setInheritedFields(DataProduct dataProduct, Fields fields) { List experts = new ArrayList<>(); for (EntityReference domainRef : domains) { - try { - Domain domain = - Entity.getEntity(DOMAIN, domainRef.getId(), "owners,experts", NON_DELETED); - owners = mergedInheritedEntityRefs(owners, domain.getOwners()); - experts = mergedInheritedEntityRefs(experts, domain.getExperts()); - } catch (EntityNotFoundException ex) { - LOG.debug( - "Skipping inherited fields from soft-deleted or missing domain {} for data product {}", - domainRef.getId(), - dataProduct.getId()); - } + Domain domain = Entity.getEntity(DOMAIN, domainRef.getId(), "owners,experts", ALL); + owners = mergedInheritedEntityRefs(owners, domain.getOwners()); + experts = mergedInheritedEntityRefs(experts, domain.getExperts()); } // inherit only if applicable and empty if (inheritOwners) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 2033b73ca488..50d19d2a8d27 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -48,7 +48,6 @@ import org.openmetadata.schema.utils.JsonUtils; import org.openmetadata.schema.utils.ResultList; import org.openmetadata.service.Entity; -import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.resources.domains.DomainResource; import org.openmetadata.service.resources.feeds.MessageParser.EntityLink; import org.openmetadata.service.search.DefaultInheritedFieldEntitySearch; @@ -187,16 +186,9 @@ public void setInheritedFields(Domain domain, Fields fields) { // domain EntityReference parentRef = domain.getParent() != null ? domain.getParent() : getParent(domain); if (parentRef != null) { - try { - Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", NON_DELETED); - inheritOwners(domain, fields, parent); - inheritExperts(domain, fields, parent); - } catch (EntityNotFoundException ex) { - LOG.debug( - "Skipping inherited fields from soft-deleted or missing parent domain {} for domain {}", - parentRef.getId(), - domain.getId()); - } + Domain parent = Entity.getEntity(DOMAIN, parentRef.getId(), "owners,experts", ALL); + inheritOwners(domain, fields, parent); + inheritExperts(domain, fields, parent); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java index b016a99f2055..847e8e22580a 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TagRepository.java @@ -15,6 +15,7 @@ import static org.openmetadata.common.utils.CommonUtil.listOrEmpty; import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Include.ALL; import static org.openmetadata.schema.type.Include.NON_DELETED; import static org.openmetadata.service.Entity.CLASSIFICATION; import static org.openmetadata.service.Entity.FIELD_CERTIFICATION; From 736b7739200fd15d1aa67e1cd42caac6ffedef3f Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 21:48:57 +0530 Subject: [PATCH 16/22] Thread Include parameter through bulk relationship resolution path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit setFieldsInBulk, fetchAndSetFields, fetchAndSetRelationshipFieldsInBulk, and resolveRelationshipEntityReferencesByType now accept an Include parameter so that include=all list queries correctly surface soft-deleted relationship targets (owners, experts, reviewers) — matching the single-entity GET path behaviour via RelationIncludes. All existing 2-param overrides delegate to the new 3-param overloads with NON_DELETED, so all subclass repositories and the update hydration path are unchanged. All list/get callers with access to an Include value (listAfter, listBefore, listAll, listAfterKeyset, listWithOffset, get, getByNames) now forward that value through the chain. --- .../service/jdbi3/EntityRepository.java | 44 +++-- .../EntityRepositoryIncludeThreadingTest.java | 173 ++++++++++++++++++ 2 files changed, 201 insertions(+), 16 deletions(-) create mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 7ba71d1e240b..3b91583fe601 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1322,7 +1322,7 @@ private void withPhase(String phaseName, Runnable operation) { public final List get(UriInfo uriInfo, List ids, Fields fields, Include include) { List entities = find(ids, include); try (var ignored = phase("setFieldsBulk")) { - setFieldsInBulk(fields, entities); + setFieldsInBulk(fields, entities, include); } entities.forEach(entity -> withHref(uriInfo, entity)); return entities; @@ -1765,7 +1765,7 @@ public final EntityReference getReferenceByName(String fqn, Include include) { public final List getByNames( UriInfo uriInfo, List entityFQNs, Fields fields, Include include) { List entities = findByNames(entityFQNs, include); - setFieldsInBulk(fields, entities); + setFieldsInBulk(fields, entities, include); entities.forEach(entity -> withHref(uriInfo, entity)); return entities; } @@ -1847,7 +1847,7 @@ public final List listAll(Fields fields, ListFilter filter) { T entity = JsonUtils.readValue(json, entityClass); entities.add(entity); } - setFieldsInBulk(fields, entities); + setFieldsInBulk(fields, entities, filter.getInclude()); return entities; } @@ -1879,11 +1879,15 @@ public final List listAllForCSV(Fields fields, String parentFqn) { * Example implementation can be found in {@link GlossaryTermRepository#setFieldsInBulk}. */ public void setFieldsInBulk(Fields fields, List entities) { + setFieldsInBulk(fields, entities, Include.NON_DELETED); + } + + public void setFieldsInBulk(Fields fields, List entities, Include include) { if (entities == null || entities.isEmpty()) { return; } try (var ignored = phase("fetchFields")) { - fetchAndSetFields(entities, fields); + fetchAndSetFields(entities, fields, include); } try (var ignored = phase("setInheritedFields")) { setInheritedFields(entities, fields); @@ -1927,7 +1931,7 @@ public ResultList listAfter( } } try (var ignored = phase("setFieldsBulk")) { - setFieldsInBulk(fields, entities); + setFieldsInBulk(fields, entities, filter.getInclude()); } entities.forEach(entity -> withHref(uriInfo, entity)); @@ -1964,7 +1968,8 @@ public ResultList listAfterKeyset( boolean hasMoreData = jsons.size() > limitParam; List jsonsToProcess = hasMoreData ? jsons.subList(0, limitParam) : jsons; - Iterator> iterator = serializeJsons(jsonsToProcess, fields, null); + Iterator> iterator = + serializeJsons(jsonsToProcess, fields, null, filter.getInclude()); while (iterator.hasNext()) { Either either = iterator.next(); if (either.right().isPresent()) { @@ -2017,7 +2022,7 @@ public ResultList listBefore( List jsons = dao.listBefore(filter, limitParam + 1, beforeName, beforeId); List entities = JsonUtils.readObjects(jsons, entityClass); - setFieldsInBulk(fields, entities); + setFieldsInBulk(fields, entities, filter.getInclude()); entities.forEach(entity -> withHref(uriInfo, entity)); int total = dao.listCount(filter); @@ -2119,7 +2124,8 @@ public final ResultList listWithOffset( String beforeOffset = getBeforeOffset(offsetInt, limitParam); if (limitParam > 0) { List jsons = callable.apply(filter, limitParam, offsetInt); - Iterator> iterator = serializeJsons(jsons, fields, uriInfo); + Iterator> iterator = + serializeJsons(jsons, fields, uriInfo, filter.getInclude()); while (iterator.hasNext()) { Either either = iterator.next(); if (either.right().isPresent()) { @@ -8421,7 +8427,12 @@ public static void validateColumn(Table table, String columnName, Boolean caseSe } protected void fetchAndSetFields(List entities, Fields fields) { - Set relationshipFieldsHandled = fetchAndSetRelationshipFieldsInBulk(entities, fields); + fetchAndSetFields(entities, fields, Include.NON_DELETED); + } + + protected void fetchAndSetFields(List entities, Fields fields, Include include) { + Set relationshipFieldsHandled = + fetchAndSetRelationshipFieldsInBulk(entities, fields, include); for (Entry, Fields>> entry : fieldFetchers.entrySet()) { if (relationshipFieldsHandled.contains(entry.getKey())) { continue; @@ -8430,7 +8441,8 @@ protected void fetchAndSetFields(List entities, Fields fields) { } } - private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields fields) { + private Set fetchAndSetRelationshipFieldsInBulk( + List entities, Fields fields, Include include) { if (nullOrEmpty(entities) || fields == null) { return Collections.emptySet(); } @@ -8497,9 +8509,9 @@ private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields .findToBatchWithRelations(entityIds, entityType, outgoingRelations, ALL); Map> incomingRefsByType = - resolveRelationshipEntityReferencesByType(incomingRecords, true); + resolveRelationshipEntityReferencesByType(incomingRecords, true, include); Map> outgoingRefsByType = - resolveRelationshipEntityReferencesByType(outgoingRecords, false); + resolveRelationshipEntityReferencesByType(outgoingRecords, false, include); Map> ownersByEntity = loadOwners ? new HashMap<>() : null; Map> followersByEntity = loadFollowers ? new HashMap<>() : null; @@ -8680,7 +8692,7 @@ private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields } private Map> resolveRelationshipEntityReferencesByType( - List records, boolean fromSide) { + List records, boolean fromSide, Include include) { if (records == null || records.isEmpty()) { return Collections.emptyMap(); } @@ -8705,7 +8717,7 @@ private Map> resolveRelationshipEntityReferen for (Entry> entry : idsByType.entrySet()) { List refs = Entity.getEntityReferencesByIdsRespectingInclude( - entry.getKey(), new ArrayList<>(entry.getValue()), Include.NON_DELETED); + entry.getKey(), new ArrayList<>(entry.getValue()), include); refsByType.put( entry.getKey(), refs.stream() @@ -9354,7 +9366,7 @@ List entityListToStrings(List entities) { } private Iterator> serializeJsons( - List jsons, Fields fields, UriInfo uriInfo) { + List jsons, Fields fields, UriInfo uriInfo, Include include) { List> results = new ArrayList<>(); List entities = new ArrayList<>(); @@ -9373,7 +9385,7 @@ private Iterator> serializeJsons( if (!entities.isEmpty()) { try { - setFieldsInBulk(fields, entities); + setFieldsInBulk(fields, entities, include); if (!nullOrEmpty(uriInfo)) { entities.forEach(entity -> withHref(uriInfo, entity)); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java new file mode 100644 index 000000000000..5debf9ef3949 --- /dev/null +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java @@ -0,0 +1,173 @@ +/* + * Copyright 2021 Collate + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * http://www.apache.org/licenses/LICENSE-2.0 + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.openmetadata.service.jdbi3; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyList; +import static org.mockito.ArgumentMatchers.anyString; +import static org.mockito.Mockito.CALLS_REAL_METHODS; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.mockStatic; +import static org.mockito.Mockito.when; + +import java.util.List; +import java.util.Set; +import java.util.UUID; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.MockedStatic; +import org.openmetadata.schema.entity.data.Pipeline; +import org.openmetadata.schema.type.Include; +import org.openmetadata.schema.type.Relationship; +import org.openmetadata.service.Entity; +import org.openmetadata.service.util.EntityUtil.Fields; +import org.openmetadata.service.util.EntityUtil.RelationIncludes; + +class EntityRepositoryIncludeThreadingTest { + + private CollectionDAO daoCollection; + private CollectionDAO.EntityRelationshipDAO relationshipDAO; + private CollectionDAO.PipelineDAO pipelineDAO; + private TestPipelineRepo repo; + + private static class TestPipelineRepo extends EntityRepository { + TestPipelineRepo(CollectionDAO.PipelineDAO dao) { + super("pipelines", Entity.PIPELINE, Pipeline.class, dao, "owners", "owners"); + } + + @Override + protected void setFields(Pipeline entity, Fields fields, RelationIncludes r) {} + + @Override + protected void clearFields(Pipeline entity, Fields fields) {} + + @Override + protected void prepare(Pipeline entity, boolean update) {} + + @Override + protected void storeEntity(Pipeline entity, boolean update) {} + + @Override + protected void storeRelationships(Pipeline entity) {} + } + + @BeforeEach + void setUp() { + daoCollection = mock(CollectionDAO.class); + relationshipDAO = mock(CollectionDAO.EntityRelationshipDAO.class); + pipelineDAO = mock(CollectionDAO.PipelineDAO.class); + + when(daoCollection.tagUsageDAO()).thenReturn(mock(CollectionDAO.TagUsageDAO.class)); + when(daoCollection.relationshipDAO()).thenReturn(relationshipDAO); + + Entity.setCollectionDAO(daoCollection); + Entity.setJobDAO(null); + Entity.setSearchRepository(null); + Entity.setEntityRelationshipRepository(null); + + repo = new TestPipelineRepo(pipelineDAO); + } + + @AfterEach + void tearDown() { + Entity.setCollectionDAO(null); + Entity.setJobDAO(null); + Entity.setSearchRepository(null); + Entity.setEntityRelationshipRepository(null); + } + + private Pipeline pipelineWithOwner(UUID pipelineId, UUID ownerId) { + CollectionDAO.EntityRelationshipObject ownerRel = + CollectionDAO.EntityRelationshipObject.builder() + .fromId(ownerId.toString()) + .toId(pipelineId.toString()) + .fromEntity(Entity.USER) + .toEntity(Entity.PIPELINE) + .relation(Relationship.OWNS.ordinal()) + .build(); + when(relationshipDAO.findFromBatchWithRelations( + anyList(), anyString(), anyList(), any(Include.class))) + .thenReturn(List.of(ownerRel)); + return new Pipeline() + .withId(pipelineId) + .withName("test-pipeline") + .withFullyQualifiedName("service.test-pipeline"); + } + + @Test + void setFieldsInBulk_twoParam_passesNonDeleted() { + Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); + Fields fields = new Fields(Set.of("owners")); + + ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); + try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { + entityStatic + .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) + .thenReturn(List.of()); + + repo.setFieldsInBulk(fields, List.of(pipeline)); + + entityStatic.verify( + () -> + Entity.getEntityReferencesByIdsRespectingInclude( + anyString(), anyList(), includeCaptor.capture())); + assertEquals(Include.NON_DELETED, includeCaptor.getValue()); + } + } + + @Test + void setFieldsInBulk_withNonDeleted_passesNonDeleted() { + Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); + Fields fields = new Fields(Set.of("owners")); + + ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); + try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { + entityStatic + .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) + .thenReturn(List.of()); + + repo.setFieldsInBulk(fields, List.of(pipeline), Include.NON_DELETED); + + entityStatic.verify( + () -> + Entity.getEntityReferencesByIdsRespectingInclude( + anyString(), anyList(), includeCaptor.capture())); + assertEquals(Include.NON_DELETED, includeCaptor.getValue()); + } + } + + @Test + void setFieldsInBulk_withAll_passesAll() { + Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); + Fields fields = new Fields(Set.of("owners")); + + ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); + try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { + entityStatic + .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) + .thenReturn(List.of()); + + repo.setFieldsInBulk(fields, List.of(pipeline), Include.ALL); + + entityStatic.verify( + () -> + Entity.getEntityReferencesByIdsRespectingInclude( + anyString(), anyList(), includeCaptor.capture())); + assertEquals(Include.ALL, includeCaptor.getValue()); + } + } +} From 007976ec14c160192d043f2c7f6bf55186adc23f Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Wed, 8 Apr 2026 22:08:54 +0530 Subject: [PATCH 17/22] Fix subclass bypass: use ThreadLocal to carry Include through virtual setFieldsInBulk dispatch The previous approach made callers invoke the 3-param setFieldsInBulk directly, which bypassed all subclass overrides of the 2-param version (GlossaryTermRepository, DatabaseRepository, TableRepository, etc. all override 2-param with essential hydration logic like populateParentAndGlossaryReferencesInBulk). This caused a regression where entities from those repos would be missing service references, parent references, and other custom bulk-hydrated fields. Fix: store the include in a static ThreadLocal before delegating to the virtual 2-param setFieldsInBulk (so all subclass overrides run), then the 2-param fetchAndSetFields reads bulkInclude.get() so it reaches fetchAndSetRelationshipFieldsInBulk with the correct value. The 3-param overload is now final to prevent further accidental overrides. --- .../service/jdbi3/EntityRepository.java | 21 +++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index 3b91583fe601..dc6ea7963606 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -360,6 +360,9 @@ public Integer load(String key) { } }); + private static final ThreadLocal bulkInclude = + ThreadLocal.withInitial(() -> Include.NON_DELETED); + private final String collectionPath; @Getter public final Class entityClass; @Getter protected final String entityType; @@ -1879,15 +1882,11 @@ public final List listAllForCSV(Fields fields, String parentFqn) { * Example implementation can be found in {@link GlossaryTermRepository#setFieldsInBulk}. */ public void setFieldsInBulk(Fields fields, List entities) { - setFieldsInBulk(fields, entities, Include.NON_DELETED); - } - - public void setFieldsInBulk(Fields fields, List entities, Include include) { if (entities == null || entities.isEmpty()) { return; } try (var ignored = phase("fetchFields")) { - fetchAndSetFields(entities, fields, include); + fetchAndSetFields(entities, fields); } try (var ignored = phase("setInheritedFields")) { setInheritedFields(entities, fields); @@ -1898,6 +1897,16 @@ public void setFieldsInBulk(Fields fields, List entities, Include include) { } } + public final void setFieldsInBulk(Fields fields, List entities, Include include) { + Include previous = bulkInclude.get(); + bulkInclude.set(include); + try { + setFieldsInBulk(fields, entities); + } finally { + bulkInclude.set(previous); + } + } + public List listAllByParentFqn(String parentFqn) { String fqnPrefixHash = FullyQualifiedName.buildHash(parentFqn); String startHash = fqnPrefixHash + ".00000000000000000000000000000000"; @@ -8427,7 +8436,7 @@ public static void validateColumn(Table table, String columnName, Boolean caseSe } protected void fetchAndSetFields(List entities, Fields fields) { - fetchAndSetFields(entities, fields, Include.NON_DELETED); + fetchAndSetFields(entities, fields, bulkInclude.get()); } protected void fetchAndSetFields(List entities, Fields fields, Include include) { From 8b443b1869a9755cf041a844170c4248c2b45c23 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Thu, 9 Apr 2026 12:00:43 +0530 Subject: [PATCH 18/22] =?UTF-8?q?Fix=20null=20Include=20in=20setFieldsInBu?= =?UTF-8?q?lk=20=E2=80=94=20default=20to=20NON=5FDELETED?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Callers like DomainResource/DataProductResource/PersonaResource build new ListFilter(null), so filter.getInclude() returns null. That null was stored in the bulkInclude ThreadLocal and propagated into getEntityReferencesByIdsRespectingInclude, bypassing the soft-delete filter for USER/TEAM types and re-introducing soft-deleted users in owners/experts/reviewers/followers on those list endpoints. Resolve null to NON_DELETED before setting the ThreadLocal. Add unit test covering null → NON_DELETED defaulting. --- .../service/jdbi3/EntityRepository.java | 3 ++- .../EntityRepositoryIncludeThreadingTest.java | 21 +++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index dc6ea7963606..3c8a91427348 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1898,8 +1898,9 @@ public void setFieldsInBulk(Fields fields, List entities) { } public final void setFieldsInBulk(Fields fields, List entities, Include include) { + Include resolved = include != null ? include : Include.NON_DELETED; Include previous = bulkInclude.get(); - bulkInclude.set(include); + bulkInclude.set(resolved); try { setFieldsInBulk(fields, entities); } finally { diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java index 5debf9ef3949..2578b38b24e5 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java @@ -170,4 +170,25 @@ void setFieldsInBulk_withAll_passesAll() { assertEquals(Include.ALL, includeCaptor.getValue()); } } + + @Test + void setFieldsInBulk_withNull_defaultsToNonDeleted() { + Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); + Fields fields = new Fields(Set.of("owners")); + + ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); + try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { + entityStatic + .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) + .thenReturn(List.of()); + + repo.setFieldsInBulk(fields, List.of(pipeline), null); + + entityStatic.verify( + () -> + Entity.getEntityReferencesByIdsRespectingInclude( + anyString(), anyList(), includeCaptor.capture())); + assertEquals(Include.NON_DELETED, includeCaptor.getValue()); + } + } } From 70a4b67fafcc1fdad5a0a13e7b72543dfff71732 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Mon, 20 Apr 2026 14:31:01 +0530 Subject: [PATCH 19/22] Fix #27107: fix inverted ternary root cause; harden bulk-list NON_DELETED MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Root cause: Entity.java had an inverted ternary in getEntityReferenceById and getEntityReferencesByIds: // WRONG — forces ALL when type supports soft delete include = repository.supportsSoftDelete ? Include.ALL : include; // CORRECT — respects caller's include; falls back to ALL only when the // type has no deleted column and filtering isn't possible include = repository.supportsSoftDelete ? include : Include.ALL; PR #25284 worked around this by introducing getEntityReferenceByIdRespectingInclude / getEntityReferencesByIdsRespectingInclude with the correct ternary, but the original broken methods were never fixed — a permanent trap. This commit fixes the root and deletes the RespectingInclude pair (now identical after the flip). For the bulk-list path (GET /entities?fields=...) nested reference hydration is hardcoded to NON_DELETED unconditionally in resolveRelationshipEntityReferencesByType and all five batchFetch* methods (owners, followers, reviewers, experts, votes). The ?include= query param controls which top-level entities appear in the list; it does not change the semantics of nested relationship pointers, which should always resolve to live entities. For single-entity GET: EntityResource.getInternal/getByNameInternal had a latent null-include bug for resources (Domain, DataProduct, EventSubscription, Query) that don't declare @QueryParam("include"). Those null values defaulted to ALL inside RelationIncludes, leaking soft-deleted nested refs. Fixed by defaulting null → NON_DELETED before constructing RelationIncludes. Remove ThreadLocal bulkInclude and the include-threading approach added earlier in this branch — superseded by the root-cause fix and the NON_DELETED semantic above. Delete EntityRepositoryIncludeThreadingTest (tested the rejected approach). Tests: 7 integration tests covering single-GET and list endpoints for Domain, DataProduct, and GlossaryTerm; verify soft-deleted experts/owners/reviewers are absent from both paths, including with ?include=all on the list endpoint. --- .../it/tests/DataProductResourceIT.java | 41 ++++ .../it/tests/DomainResourceIT.java | 49 +++++ .../java/org/openmetadata/service/Entity.java | 46 +---- .../service/jdbi3/DataProductRepository.java | 4 +- .../service/jdbi3/DomainRepository.java | 4 +- .../jdbi3/EntityRelationshipRepository.java | 5 +- .../service/jdbi3/EntityRepository.java | 44 ++-- .../EntityRepositoryIncludeThreadingTest.java | 194 ------------------ 8 files changed, 110 insertions(+), 277 deletions(-) delete mode 100644 openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java index d019280a93e6..d12b2750f296 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DataProductResourceIT.java @@ -2952,4 +2952,45 @@ void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { listed.getExperts() == null || listed.getExperts().isEmpty(), "Soft-deleted expert must not appear in list endpoint"); } + + @Test + void softDeletedOwner_notReturnedInListEndpoint(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + Domain domain = getOrCreateDomain(ns); + + String userName = ns.shortPrefix("owner_list_user"); + User owner = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName + "@test.openmetadata.org") + .withDescription("Owner user for soft-delete list test")); + + CreateDataProduct create = + new CreateDataProduct() + .withName(ns.prefix("dp_softdel_owner_list")) + .withDescription("DataProduct for soft-delete owner list test") + .withDomains(List.of(domain.getFullyQualifiedName())) + .withOwners(List.of(owner.getEntityReference())); + DataProduct dp = createEntity(create); + + client.users().delete(owner.getId().toString()); + + ListParams params = + new ListParams() + .setFields("owners") + .withDomain(domain.getFullyQualifiedName()) + .withLimit(100); + ListResponse list = client.dataProducts().list(params); + DataProduct listed = + list.getData().stream() + .filter(p -> p.getId().equals(dp.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("DataProduct not found in list")); + assertTrue( + listed.getOwners() == null || listed.getOwners().isEmpty(), + "Soft-deleted owner must not appear in list endpoint"); + } } diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java index babcfc25d500..f5d50fa03a26 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java @@ -1233,4 +1233,53 @@ void softDeletedExpert_notReturnedInListEndpoint(TestNamespace ns) { listed.getExperts() == null || listed.getExperts().isEmpty(), "Soft-deleted expert must not appear in list endpoint"); } + + @Test + void softDeletedExpert_notReturnedInListWithIncludeAll(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + + String userName = ns.shortPrefix("domain_expert_all"); + User expert = + client + .users() + .create( + new CreateUser() + .withName(userName) + .withEmail(userName + "@test.openmetadata.org") + .withDescription("Expert user for domain include-all soft-delete test")); + + CreateDomain create = + new CreateDomain() + .withName(ns.prefix("domain_softdel_all")) + .withDomainType(DomainType.AGGREGATE) + .withExperts(List.of(expert.getFullyQualifiedName())) + .withDescription("Domain for include-all soft-delete expert test"); + Domain domain = createEntity(create); + + client.users().delete(expert.getId().toString()); + + Domain listed = null; + ListParams params = + new ListParams().setFields("experts").withLimit(100).addFilter("include", "all"); + while (listed == null) { + ListResponse page = listEntities(params); + listed = + page.getData().stream() + .filter(d -> d.getId().equals(domain.getId())) + .findFirst() + .orElse(null); + String after = page.getPaging() != null ? page.getPaging().getAfter() : null; + if (listed != null || after == null) break; + params = + new ListParams() + .setFields("experts") + .withLimit(100) + .addFilter("include", "all") + .setAfter(after); + } + assertNotNull(listed, "Domain not found in list with include=all"); + assertTrue( + listed.getExperts() == null || listed.getExperts().isEmpty(), + "Soft-deleted expert must not appear even when include=all (applies to top-level only)"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java index 497661f4b989..5fe2a534efc3 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java @@ -491,7 +491,7 @@ public static EntityReference getEntityReferenceById( // For regular entities, use the standard repository EntityRepository repository = getEntityRepository(entityType); - include = repository.supportsSoftDelete ? Include.ALL : include; + include = repository.supportsSoftDelete ? include : Include.ALL; return repository.getReference(id, include); } @@ -512,7 +512,7 @@ public static List getEntityReferencesByIds( // For regular entities, use the standard repository EntityRepository repository = getEntityRepository(entityType); - include = repository.supportsSoftDelete ? Include.ALL : include; + include = repository.supportsSoftDelete ? include : Include.ALL; return repository.getReferences(ids, include); } @@ -525,48 +525,6 @@ public static EntityReference getEntityReferenceByName( return repository.getReferenceByName(fqn, include); } - /** - * Get entity reference by ID, respecting the include parameter for soft-delete filtering. Unlike - * {@link #getEntityReferenceById}, this method does NOT override the include parameter to ALL for - * repositories that support soft delete. - */ - public static EntityReference getEntityReferenceByIdRespectingInclude( - @NonNull String entityType, @NonNull UUID id, Include include) { - if (ENTITY_TS_REPOSITORY_MAP.containsKey(entityType)) { - return new EntityReference() - .withId(id) - .withType(entityType) - .withFullyQualifiedName(entityType + "." + id); - } - EntityRepository repository = getEntityRepository(entityType); - // If repository doesn't support soft delete, use ALL since there's no deleted column - include = repository.supportsSoftDelete ? include : Include.ALL; - return repository.getReference(id, include); - } - - /** - * Get entity references by IDs, respecting the include parameter for soft-delete filtering. - * Unlike {@link #getEntityReferencesByIds}, this method does NOT override the include parameter - * to ALL for repositories that support soft delete. - */ - public static List getEntityReferencesByIdsRespectingInclude( - @NonNull String entityType, @NonNull List ids, Include include) { - if (ENTITY_TS_REPOSITORY_MAP.containsKey(entityType)) { - return ids.stream() - .map( - id -> - new EntityReference() - .withId(id) - .withType(entityType) - .withFullyQualifiedName(entityType + "." + id)) - .collect(Collectors.toList()); - } - EntityRepository repository = getEntityRepository(entityType); - // If repository doesn't support soft delete, use ALL since there's no deleted column - include = repository.supportsSoftDelete ? include : Include.ALL; - return repository.getReferences(ids, include); - } - public static List getOwners(@NonNull EntityReference reference) { EntityRepository repository = getEntityRepository(reference.getType()); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java index 9ea88de01fcd..1d0d26db6d64 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DataProductRepository.java @@ -981,9 +981,7 @@ private Map> batchFetchExperts(List dat List expertIds = records.stream().map(r -> UUID.fromString(r.getToId())).distinct().toList(); Map expertRefsById = - Entity.getEntityReferencesByIdsRespectingInclude( - Entity.USER, expertIds, Include.NON_DELETED) - .stream() + Entity.getEntityReferencesByIds(Entity.USER, expertIds, Include.NON_DELETED).stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); for (CollectionDAO.EntityRelationshipObject record : records) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java index 50d19d2a8d27..244eea25bc42 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/DomainRepository.java @@ -591,9 +591,7 @@ private Map> batchFetchExperts(List domains) List expertIds = records.stream().map(r -> UUID.fromString(r.getToId())).distinct().toList(); Map expertRefsById = - Entity.getEntityReferencesByIdsRespectingInclude( - Entity.USER, expertIds, Include.NON_DELETED) - .stream() + Entity.getEntityReferencesByIds(Entity.USER, expertIds, Include.NON_DELETED).stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); records.forEach( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java index b24110afadbe..16acb2b27f4b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRelationshipRepository.java @@ -78,8 +78,7 @@ public List getEntityReferences( queryCount++; try { - List typeRefs = - Entity.getEntityReferencesByIdsRespectingInclude(entityType, ids, include); + List typeRefs = Entity.getEntityReferencesByIds(entityType, ids, include); refs.addAll(typeRefs); } catch (Exception e) { // Fallback for partial failures - fetch individually to handle deleted entities gracefully @@ -89,7 +88,7 @@ public List getEntityReferences( e.getMessage()); for (UUID id : ids) { try { - refs.add(Entity.getEntityReferenceByIdRespectingInclude(entityType, id, include)); + refs.add(Entity.getEntityReferenceById(entityType, id, include)); } catch (EntityNotFoundException ex) { // Skip deleted or missing entities skippedCount++; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index efda7a06e2f2..5f43a21eed64 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -360,9 +360,6 @@ public Integer load(String key) { } }); - private static final ThreadLocal bulkInclude = - ThreadLocal.withInitial(() -> Include.NON_DELETED); - private final String collectionPath; @Getter public final Class entityClass; @Getter protected final String entityType; @@ -1912,14 +1909,7 @@ public void setFieldsInBulk(Fields fields, List entities) { } public final void setFieldsInBulk(Fields fields, List entities, Include include) { - Include resolved = include != null ? include : Include.NON_DELETED; - Include previous = bulkInclude.get(); - bulkInclude.set(resolved); - try { - setFieldsInBulk(fields, entities); - } finally { - bulkInclude.set(previous); - } + setFieldsInBulk(fields, entities); } public List listAllByParentFqn(String parentFqn) { @@ -8497,12 +8487,7 @@ public static void validateColumn(Table table, String columnName, Boolean caseSe } protected void fetchAndSetFields(List entities, Fields fields) { - fetchAndSetFields(entities, fields, bulkInclude.get()); - } - - protected void fetchAndSetFields(List entities, Fields fields, Include include) { - Set relationshipFieldsHandled = - fetchAndSetRelationshipFieldsInBulk(entities, fields, include); + Set relationshipFieldsHandled = fetchAndSetRelationshipFieldsInBulk(entities, fields); for (Entry, Fields>> entry : fieldFetchers.entrySet()) { if (relationshipFieldsHandled.contains(entry.getKey())) { continue; @@ -8511,8 +8496,7 @@ protected void fetchAndSetFields(List entities, Fields fields, Include includ } } - private Set fetchAndSetRelationshipFieldsInBulk( - List entities, Fields fields, Include include) { + private Set fetchAndSetRelationshipFieldsInBulk(List entities, Fields fields) { if (nullOrEmpty(entities) || fields == null) { return Collections.emptySet(); } @@ -8579,9 +8563,9 @@ private Set fetchAndSetRelationshipFieldsInBulk( .findToBatchWithRelations(entityIds, entityType, outgoingRelations, ALL); Map> incomingRefsByType = - resolveRelationshipEntityReferencesByType(incomingRecords, true, include); + resolveRelationshipEntityReferencesByType(incomingRecords, true); Map> outgoingRefsByType = - resolveRelationshipEntityReferencesByType(outgoingRecords, false, include); + resolveRelationshipEntityReferencesByType(outgoingRecords, false); Map> ownersByEntity = loadOwners ? new HashMap<>() : null; Map> followersByEntity = loadFollowers ? new HashMap<>() : null; @@ -8762,7 +8746,7 @@ private Set fetchAndSetRelationshipFieldsInBulk( } private Map> resolveRelationshipEntityReferencesByType( - List records, boolean fromSide, Include include) { + List records, boolean fromSide) { if (records == null || records.isEmpty()) { return Collections.emptyMap(); } @@ -8786,8 +8770,8 @@ private Map> resolveRelationshipEntityReferen Map> refsByType = new HashMap<>(); for (Entry> entry : idsByType.entrySet()) { List refs = - Entity.getEntityReferencesByIdsRespectingInclude( - entry.getKey(), new ArrayList<>(entry.getValue()), include); + Entity.getEntityReferencesByIds( + entry.getKey(), new ArrayList<>(entry.getValue()), NON_DELETED); refsByType.put( entry.getKey(), refs.stream() @@ -9000,8 +8984,7 @@ private Map> batchFetchOwners(List entities) { ownerIdsByType.forEach( (entityType, ownerIds) -> { var ownerRefs = - Entity.getEntityReferencesByIdsRespectingInclude( - entityType, new ArrayList<>(ownerIds), NON_DELETED); + Entity.getEntityReferencesByIds(entityType, new ArrayList<>(ownerIds), NON_DELETED); var refMap = ownerRefs.stream() .collect(Collectors.toMap(EntityReference::getId, ref -> ref, (a, b) -> a)); @@ -9046,7 +9029,7 @@ private Map> batchFetchFollowers(List entities) { .collect(Collectors.toList()); Map followerRefs = - Entity.getEntityReferencesByIds(USER, followerIds, ALL).stream() + Entity.getEntityReferencesByIds(USER, followerIds, NON_DELETED).stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity())); records.forEach( @@ -9090,7 +9073,8 @@ private Map batchFetchVotes(List entities) { upVoterIds.values().forEach(allUserIds::addAll); downVoterIds.values().forEach(allUserIds::addAll); Map userRefs = - Entity.getEntityReferencesByIds(Entity.USER, new ArrayList<>(allUserIds), ALL).stream() + Entity.getEntityReferencesByIds(Entity.USER, new ArrayList<>(allUserIds), NON_DELETED) + .stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity())); for (T entity : entities) { @@ -9286,7 +9270,7 @@ private Map> batchFetchReviewers(List entities) { reviewerIdsByType.forEach( (entityType, reviewerIds) -> { var reviewerRefs = - Entity.getEntityReferencesByIdsRespectingInclude( + Entity.getEntityReferencesByIds( entityType, new ArrayList<>(reviewerIds), NON_DELETED); var refMap = reviewerRefs.stream() @@ -9371,7 +9355,7 @@ private Map> batchFetchExperts(List entities) { // Batch fetch all expert references, filtering out soft-deleted users Map expertRefs = - Entity.getEntityReferencesByIdsRespectingInclude(USER, expertIds, NON_DELETED).stream() + Entity.getEntityReferencesByIds(USER, expertIds, NON_DELETED).stream() .collect(Collectors.toMap(EntityReference::getId, Function.identity(), (a, b) -> a)); // Group experts by entity diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java deleted file mode 100644 index 2578b38b24e5..000000000000 --- a/openmetadata-service/src/test/java/org/openmetadata/service/jdbi3/EntityRepositoryIncludeThreadingTest.java +++ /dev/null @@ -1,194 +0,0 @@ -/* - * Copyright 2021 Collate - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.openmetadata.service.jdbi3; - -import static org.junit.jupiter.api.Assertions.assertEquals; -import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyList; -import static org.mockito.ArgumentMatchers.anyString; -import static org.mockito.Mockito.CALLS_REAL_METHODS; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.mockStatic; -import static org.mockito.Mockito.when; - -import java.util.List; -import java.util.Set; -import java.util.UUID; -import org.junit.jupiter.api.AfterEach; -import org.junit.jupiter.api.BeforeEach; -import org.junit.jupiter.api.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.MockedStatic; -import org.openmetadata.schema.entity.data.Pipeline; -import org.openmetadata.schema.type.Include; -import org.openmetadata.schema.type.Relationship; -import org.openmetadata.service.Entity; -import org.openmetadata.service.util.EntityUtil.Fields; -import org.openmetadata.service.util.EntityUtil.RelationIncludes; - -class EntityRepositoryIncludeThreadingTest { - - private CollectionDAO daoCollection; - private CollectionDAO.EntityRelationshipDAO relationshipDAO; - private CollectionDAO.PipelineDAO pipelineDAO; - private TestPipelineRepo repo; - - private static class TestPipelineRepo extends EntityRepository { - TestPipelineRepo(CollectionDAO.PipelineDAO dao) { - super("pipelines", Entity.PIPELINE, Pipeline.class, dao, "owners", "owners"); - } - - @Override - protected void setFields(Pipeline entity, Fields fields, RelationIncludes r) {} - - @Override - protected void clearFields(Pipeline entity, Fields fields) {} - - @Override - protected void prepare(Pipeline entity, boolean update) {} - - @Override - protected void storeEntity(Pipeline entity, boolean update) {} - - @Override - protected void storeRelationships(Pipeline entity) {} - } - - @BeforeEach - void setUp() { - daoCollection = mock(CollectionDAO.class); - relationshipDAO = mock(CollectionDAO.EntityRelationshipDAO.class); - pipelineDAO = mock(CollectionDAO.PipelineDAO.class); - - when(daoCollection.tagUsageDAO()).thenReturn(mock(CollectionDAO.TagUsageDAO.class)); - when(daoCollection.relationshipDAO()).thenReturn(relationshipDAO); - - Entity.setCollectionDAO(daoCollection); - Entity.setJobDAO(null); - Entity.setSearchRepository(null); - Entity.setEntityRelationshipRepository(null); - - repo = new TestPipelineRepo(pipelineDAO); - } - - @AfterEach - void tearDown() { - Entity.setCollectionDAO(null); - Entity.setJobDAO(null); - Entity.setSearchRepository(null); - Entity.setEntityRelationshipRepository(null); - } - - private Pipeline pipelineWithOwner(UUID pipelineId, UUID ownerId) { - CollectionDAO.EntityRelationshipObject ownerRel = - CollectionDAO.EntityRelationshipObject.builder() - .fromId(ownerId.toString()) - .toId(pipelineId.toString()) - .fromEntity(Entity.USER) - .toEntity(Entity.PIPELINE) - .relation(Relationship.OWNS.ordinal()) - .build(); - when(relationshipDAO.findFromBatchWithRelations( - anyList(), anyString(), anyList(), any(Include.class))) - .thenReturn(List.of(ownerRel)); - return new Pipeline() - .withId(pipelineId) - .withName("test-pipeline") - .withFullyQualifiedName("service.test-pipeline"); - } - - @Test - void setFieldsInBulk_twoParam_passesNonDeleted() { - Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); - Fields fields = new Fields(Set.of("owners")); - - ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); - try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { - entityStatic - .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) - .thenReturn(List.of()); - - repo.setFieldsInBulk(fields, List.of(pipeline)); - - entityStatic.verify( - () -> - Entity.getEntityReferencesByIdsRespectingInclude( - anyString(), anyList(), includeCaptor.capture())); - assertEquals(Include.NON_DELETED, includeCaptor.getValue()); - } - } - - @Test - void setFieldsInBulk_withNonDeleted_passesNonDeleted() { - Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); - Fields fields = new Fields(Set.of("owners")); - - ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); - try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { - entityStatic - .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) - .thenReturn(List.of()); - - repo.setFieldsInBulk(fields, List.of(pipeline), Include.NON_DELETED); - - entityStatic.verify( - () -> - Entity.getEntityReferencesByIdsRespectingInclude( - anyString(), anyList(), includeCaptor.capture())); - assertEquals(Include.NON_DELETED, includeCaptor.getValue()); - } - } - - @Test - void setFieldsInBulk_withAll_passesAll() { - Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); - Fields fields = new Fields(Set.of("owners")); - - ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); - try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { - entityStatic - .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) - .thenReturn(List.of()); - - repo.setFieldsInBulk(fields, List.of(pipeline), Include.ALL); - - entityStatic.verify( - () -> - Entity.getEntityReferencesByIdsRespectingInclude( - anyString(), anyList(), includeCaptor.capture())); - assertEquals(Include.ALL, includeCaptor.getValue()); - } - } - - @Test - void setFieldsInBulk_withNull_defaultsToNonDeleted() { - Pipeline pipeline = pipelineWithOwner(UUID.randomUUID(), UUID.randomUUID()); - Fields fields = new Fields(Set.of("owners")); - - ArgumentCaptor includeCaptor = ArgumentCaptor.forClass(Include.class); - try (MockedStatic entityStatic = mockStatic(Entity.class, CALLS_REAL_METHODS)) { - entityStatic - .when(() -> Entity.getEntityReferencesByIdsRespectingInclude(any(), any(), any())) - .thenReturn(List.of()); - - repo.setFieldsInBulk(fields, List.of(pipeline), null); - - entityStatic.verify( - () -> - Entity.getEntityReferencesByIdsRespectingInclude( - anyString(), anyList(), includeCaptor.capture())); - assertEquals(Include.NON_DELETED, includeCaptor.getValue()); - } - } -} From 7217773b8bc76ba7507300c27d0f2e0f579f3ce3 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Mon, 20 Apr 2026 15:33:18 +0530 Subject: [PATCH 20/22] Fix Copilot review: null-guard in batchFetchFollowers, remove dead 3-arg setFieldsInBulk, add follower/voter IT tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - batchFetchFollowers: add null-check before adding followerRef to list. When a follower is soft-deleted, getEntityReferencesByIds(NON_DELETED) omits their ID from the result map, so followerRefs.get(followerId) returns null. All other batchFetch* methods already guard this; batchFetchFollowers was the only one missing the check. - Remove the dead 3-arg setFieldsInBulk(Fields, List, Include) overload that accepted include but silently discarded it, delegating to the 2-arg form. Revert all 6 call sites back to the 2-arg form. The overload was misleading — callers appeared to plumb include through but bulk hydration was never affected; NON_DELETED is hardcoded in resolveRelationship- EntityReferencesByType. - Add integration tests for followers and votes: softDeletedFollower_notReturnedInListEndpoint and softDeletedVoter_notReturnedInListEndpoint in DomainResourceIT. Both pass (5/5 DomainResourceIT soft-delete tests green). --- .../it/tests/DomainResourceIT.java | 77 +++++++++++++++++++ .../service/jdbi3/EntityRepository.java | 20 +++-- 2 files changed, 86 insertions(+), 11 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java index f5d50fa03a26..91fc648a3cad 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java @@ -29,16 +29,20 @@ import org.junit.jupiter.api.parallel.ExecutionMode; import org.openmetadata.it.util.SdkClients; import org.openmetadata.it.util.TestNamespace; +import org.openmetadata.schema.api.VoteRequest; import org.openmetadata.schema.api.domains.CreateDomain; import org.openmetadata.schema.api.domains.CreateDomain.DomainType; import org.openmetadata.schema.api.teams.CreateUser; import org.openmetadata.schema.entity.domains.Domain; import org.openmetadata.schema.entity.teams.User; +import org.openmetadata.schema.type.ChangeEvent; import org.openmetadata.schema.type.EntityHistory; import org.openmetadata.schema.type.EntityReference; +import org.openmetadata.schema.type.Votes; import org.openmetadata.sdk.client.OpenMetadataClient; import org.openmetadata.sdk.models.ListParams; import org.openmetadata.sdk.models.ListResponse; +import org.openmetadata.sdk.network.HttpMethod; /** * Integration tests for Domain entity operations. @@ -1282,4 +1286,77 @@ void softDeletedExpert_notReturnedInListWithIncludeAll(TestNamespace ns) { listed.getExperts() == null || listed.getExperts().isEmpty(), "Soft-deleted expert must not appear even when include=all (applies to top-level only)"); } + + @Test + void softDeletedFollower_notReturnedInListEndpoint(TestNamespace ns) { + OpenMetadataClient client = SdkClients.adminClient(); + + String userName = ns.shortPrefix("follower_list"); + User follower = + client + .users() + .create( + new CreateUser().withName(userName).withEmail(userName + "@test.openmetadata.org")); + + Domain domain = createEntity(createRequest(ns.prefix("dom_follower"), ns)); + + client + .getHttpClient() + .execute( + HttpMethod.PUT, + "/v1/domains/" + domain.getId() + "/followers", + follower.getId(), + ChangeEvent.class); + + client.users().delete(follower.getId().toString()); + + ListParams params = new ListParams().setFields("followers").withLimit(100); + ListResponse list = listEntities(params); + Domain listed = + list.getData().stream() + .filter(d -> d.getId().equals(domain.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("Domain not found in list")); + assertTrue( + listed.getFollowers() == null || listed.getFollowers().isEmpty(), + "Soft-deleted follower must not appear in list endpoint"); + } + + @Test + void softDeletedVoter_notReturnedInListEndpoint(TestNamespace ns) { + String userName = ns.shortPrefix("voter_list"); + String userEmail = userName + "@test.openmetadata.org"; + + OpenMetadataClient adminClient = SdkClients.adminClient(); + User voter = + adminClient.users().create(new CreateUser().withName(userName).withEmail(userEmail)); + + Domain domain = createEntity(createRequest(ns.prefix("dom_voter"), ns)); + + OpenMetadataClient voterClient = SdkClients.createClient(userEmail, userEmail, new String[] {}); + voterClient + .getHttpClient() + .execute( + HttpMethod.PUT, + "/v1/domains/" + domain.getId() + "/vote", + new VoteRequest().withUpdatedVoteType(VoteRequest.VoteType.VOTED_UP), + ChangeEvent.class); + + adminClient.users().delete(voter.getId().toString()); + + ListParams params = new ListParams().setFields("votes").withLimit(100); + ListResponse list = listEntities(params); + Domain listed = + list.getData().stream() + .filter(d -> d.getId().equals(domain.getId())) + .findFirst() + .orElseThrow(() -> new AssertionError("Domain not found in list")); + Votes votes = listed.getVotes(); + boolean voterInUpVotes = + votes != null + && votes.getUpVoters() != null + && votes.getUpVoters().stream() + .anyMatch(ref -> ref != null && voter.getId().equals(ref.getId())); + assertFalse(voterInUpVotes, "Soft-deleted voter must not appear in list endpoint votes"); + } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index b62f9bd34a37..c3c9e66a6fe5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1383,7 +1383,7 @@ private void withPhase(String phaseName, Runnable operation) { public final List get(UriInfo uriInfo, List ids, Fields fields, Include include) { List entities = find(ids, include); try (var ignored = phase("setFieldsBulk")) { - setFieldsInBulk(fields, entities, include); + setFieldsInBulk(fields, entities); } entities.forEach(entity -> withHref(uriInfo, entity)); return entities; @@ -1826,7 +1826,7 @@ public final EntityReference getReferenceByName(String fqn, Include include) { public final List getByNames( UriInfo uriInfo, List entityFQNs, Fields fields, Include include) { List entities = findByNames(entityFQNs, include); - setFieldsInBulk(fields, entities, include); + setFieldsInBulk(fields, entities); entities.forEach(entity -> withHref(uriInfo, entity)); return entities; } @@ -1908,7 +1908,7 @@ public final List listAll(Fields fields, ListFilter filter) { T entity = JsonUtils.readValue(json, entityClass); entities.add(entity); } - setFieldsInBulk(fields, entities, filter.getInclude()); + setFieldsInBulk(fields, entities); return entities; } @@ -1955,10 +1955,6 @@ public void setFieldsInBulk(Fields fields, List entities) { } } - public final void setFieldsInBulk(Fields fields, List entities, Include include) { - setFieldsInBulk(fields, entities); - } - public List listAllByParentFqn(String parentFqn) { String fqnPrefixHash = FullyQualifiedName.buildHash(parentFqn); String startHash = fqnPrefixHash + ".00000000000000000000000000000000"; @@ -2019,7 +2015,7 @@ private List listInternal( entities = JsonUtils.readObjects(jsons, entityClass); } try (var ignored = phase("setFieldsBulk")) { - setFieldsInBulk(fields, entities, include); + setFieldsInBulk(fields, entities); } entities.forEach(entity -> withHref(uriInfo, entity)); return entities; @@ -2097,7 +2093,7 @@ public ResultList listBefore( List jsons = dao.listBefore(filter, limitParam + 1, beforeName, beforeId); List entities = JsonUtils.readObjects(jsons, entityClass); - setFieldsInBulk(fields, entities, filter.getInclude()); + setFieldsInBulk(fields, entities); entities.forEach(entity -> withHref(uriInfo, entity)); int total = dao.listCount(filter); @@ -9089,7 +9085,9 @@ record -> { UUID entityId = UUID.fromString(record.getToId()); UUID followerId = UUID.fromString(record.getFromId()); EntityReference followerRef = followerRefs.get(followerId); - followersMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(followerRef); + if (followerRef != null) { + followersMap.computeIfAbsent(entityId, k -> new ArrayList<>()).add(followerRef); + } }); return followersMap; @@ -9491,7 +9489,7 @@ private Iterator> serializeJsons( if (!entities.isEmpty()) { try { - setFieldsInBulk(fields, entities, include); + setFieldsInBulk(fields, entities); if (!nullOrEmpty(uriInfo)) { entities.forEach(entity -> withHref(uriInfo, entity)); } From 135be1cd127db0ee00e3f6d6cfac1ddb83719861 Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Mon, 20 Apr 2026 17:47:20 +0530 Subject: [PATCH 21/22] fix: bump list limit to 1000000 in follower/voter IT tests to handle large CI datasets --- .../test/java/org/openmetadata/it/tests/DomainResourceIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java index 91fc648a3cad..793bb80e1ad2 100644 --- a/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java +++ b/openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/DomainResourceIT.java @@ -1310,7 +1310,7 @@ void softDeletedFollower_notReturnedInListEndpoint(TestNamespace ns) { client.users().delete(follower.getId().toString()); - ListParams params = new ListParams().setFields("followers").withLimit(100); + ListParams params = new ListParams().setFields("followers").withLimit(1000000); ListResponse list = listEntities(params); Domain listed = list.getData().stream() @@ -1344,7 +1344,7 @@ void softDeletedVoter_notReturnedInListEndpoint(TestNamespace ns) { adminClient.users().delete(voter.getId().toString()); - ListParams params = new ListParams().setFields("votes").withLimit(100); + ListParams params = new ListParams().setFields("votes").withLimit(1000000); ListResponse list = listEntities(params); Domain listed = list.getData().stream() From 607743bc83c914d185c17cdb1d7a34105f8f8ece Mon Sep 17 00:00:00 2001 From: Ram Narayan Balaji Date: Mon, 20 Apr 2026 18:36:12 +0530 Subject: [PATCH 22/22] fix: remove unused Include param from listInternal and serializeJsons --- .../service/jdbi3/EntityRepository.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java index c3c9e66a6fe5..a38781138f2b 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/EntityRepository.java @@ -1981,7 +1981,7 @@ public ResultList listAfter( String afterId = cursorMap.get("id"); List jsons = dao.listAfter(filter, limitParam + 1, afterName, afterId); - entities = listInternal(jsons, fields, uriInfo, filter.getInclude()); + entities = listInternal(jsons, fields, uriInfo); String beforeCursor; String afterCursor = null; @@ -2003,13 +2003,12 @@ public ResultList listAfterWithOffset( int total = dao.listCount(filter); List jsons = dao.listAfter(filter, limit, offset); - List entities = listInternal(jsons, fields, uriInfo, filter.getInclude()); + List entities = listInternal(jsons, fields, uriInfo); return new ResultList<>(entities, offset, limit, total); } - private List listInternal( - List jsons, Fields fields, UriInfo uriInfo, Include include) { + private List listInternal(List jsons, Fields fields, UriInfo uriInfo) { List entities; try (var ignored = phase("jsonDeserialize")) { entities = JsonUtils.readObjects(jsons, entityClass); @@ -2039,8 +2038,7 @@ public ResultList listAfterKeyset( boolean hasMoreData = jsons.size() > limitParam; List jsonsToProcess = hasMoreData ? jsons.subList(0, limitParam) : jsons; - Iterator> iterator = - serializeJsons(jsonsToProcess, fields, null, filter.getInclude()); + Iterator> iterator = serializeJsons(jsonsToProcess, fields, null); while (iterator.hasNext()) { Either either = iterator.next(); if (either.right().isPresent()) { @@ -2195,8 +2193,7 @@ public final ResultList listWithOffset( String beforeOffset = getBeforeOffset(offsetInt, limitParam); if (limitParam > 0) { List jsons = callable.apply(filter, limitParam, offsetInt); - Iterator> iterator = - serializeJsons(jsons, fields, uriInfo, filter.getInclude()); + Iterator> iterator = serializeJsons(jsons, fields, uriInfo); while (iterator.hasNext()) { Either either = iterator.next(); if (either.right().isPresent()) { @@ -9470,7 +9467,7 @@ List entityListToStrings(List entities) { } private Iterator> serializeJsons( - List jsons, Fields fields, UriInfo uriInfo, Include include) { + List jsons, Fields fields, UriInfo uriInfo) { List> results = new ArrayList<>(); List entities = new ArrayList<>();