fix(#25764): Implement UTF-8 encoding standardization for CSV import/…

.gitignore

@@ -90,6 +90,7 @@ openmetadata-ui/src/main/resources/ui/.env
 openmetadata-ui/src/main/resources/ui/playwright/.auth
 openmetadata-ui/src/main/resources/ui/blob-report
 openmetadata-ui/src/main/resources/ui/test-results/
+openmetadata-ui/src/main/resources/ui/debug.json
 
 #UI - Dereferenced Schemas
 openmetadata-ui/src/main/resources/ui/src/jsons/*

openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/BaseEntityIT.java

@@ -5636,6 +5636,45 @@ void test_importExportRoundTrip(TestNamespace ns) {
     }
   }
 
+  /**
+   * Test: CSV import/export round-trip preserves Unicode text.
+   * Verifies non-ASCII content survives export, import, and re-export.
+   */
+  @Test
+  void test_importExportRoundTripUnicode(TestNamespace ns) {
+    Assumptions.assumeTrue(supportsImportExport, "Entity does not support import/export");
+
+    org.openmetadata.sdk.services.EntityServiceBase<T> service = getEntityService();
+    Assumptions.assumeTrue(service != null, "Entity service not provided");
+
+    String containerName = getImportExportContainerName(ns);
+    Assumptions.assumeTrue(containerName != null, "Container name not provided");
+
+    K createRequest = createMinimalRequest(ns);
+    String unicodeDescription = "中文描述 - CSV import/export round trip";
+    setDescription(createRequest, unicodeDescription);
+    T entity = createEntity(createRequest);
+    assertNotNull(entity, "Entity should be created");
+
+    try {
+      String exportedCsv = service.exportCsv(containerName);
+      assertNotNull(exportedCsv, "Export should return CSV data");
+      assertTrue(
+          exportedCsv.contains(unicodeDescription), "Exported CSV should contain Unicode text");
+
+      CsvImportResult importResult = performImportCsv(ns, exportedCsv, false);
+      assertEquals(
+          ApiStatus.SUCCESS, importResult.getStatus(), "Unicode round-trip should succeed");
+
+      String reExportedCsv = service.exportCsv(containerName);
+      assertNotNull(reExportedCsv, "Re-export should return CSV data");
+      assertTrue(
+          reExportedCsv.contains(unicodeDescription), "Re-exported CSV should retain Unicode text");
+    } catch (Exception e) {
+      fail("Unicode import/export round-trip failed: " + e.getMessage());
+    }
+  }
+
   // ===================================================================
   // COMPREHENSIVE CSV IMPORT/EXPORT TESTS
   // Template-based tests that work with any entity CSV structure

openmetadata-service/src/main/java/org/openmetadata/csv/CsvUtil.java

@@ -40,6 +40,7 @@
 public final class CsvUtil {
   public static final String SEPARATOR = ",";
   public static final String FIELD_SEPARATOR = ";";
+  public static final String UTF8_BOM = "\uFEFF";
 
   public static final String ENTITY_TYPE_SEPARATOR = ":";
   public static final String LINE_SEPARATOR = "\r\n";
@@ -50,6 +51,13 @@ private CsvUtil() {
     // Utility class hides the constructor
   }
 
+  public static String stripUtf8Bom(String value) {
+    if (value == null || value.isEmpty() || !value.startsWith(UTF8_BOM)) {
+      return value;
+    }
+    return value.substring(1);
+  }
+
   public static String formatCsv(CsvFile csvFile) throws IOException {
     // CSV file is generated by the backend and the data exported is expected to be correct. Hence,
     // no validation

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ColumnRepository.java

@@ -37,6 +37,7 @@
 import java.util.function.BiConsumer;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.csv.CsvUtil;
 import org.openmetadata.schema.EntityInterface;
 import org.openmetadata.schema.api.data.BulkColumnUpdatePreview;
 import org.openmetadata.schema.api.data.BulkColumnUpdateRequest;
@@ -875,7 +876,7 @@ public CsvImportResult importColumnsCSV(
     result.setNumberOfRowsPassed(0);
     result.setNumberOfRowsFailed(0);
 
-    String[] lines = csv.split("\n");
+    String[] lines = CsvUtil.stripUtf8Bom(csv).split("\n");
     if (lines.length <= 1) {
       result.setStatus(ApiStatus.ABORTED);
       result.setAbortReason("No data to import");

openmetadata-service/src/main/java/org/openmetadata/service/resources/EntityResource.java

@@ -53,6 +53,7 @@
 import lombok.extern.slf4j.Slf4j;
 import org.openmetadata.csv.CsvExportProgressCallback;
 import org.openmetadata.csv.CsvImportProgressCallback;
+import org.openmetadata.csv.CsvUtil;
 import org.openmetadata.schema.BulkAssetsRequestInterface;
 import org.openmetadata.schema.CreateEntity;
 import org.openmetadata.schema.EntityInterface;
@@ -1005,18 +1006,19 @@ protected CsvImportResult importCsvInternal(
     OperationContext operationContext =
         new OperationContext(entityType, MetadataOperation.EDIT_ALL);
     authorizer.authorize(securityContext, operationContext, getResourceContextByName(name));
+    String normalizedCsv = CsvUtil.stripUtf8Bom(csv);
     CsvImportResult result =
         nullOrEmpty(versioningEntityType)
             ? repository.importFromCsv(
                 name,
-                csv,
+                normalizedCsv,
                 dryRun,
                 securityContext.getUserPrincipal().getName(),
                 recursive,
                 progressCallback)
             : repository.importFromCsv(
                 name,
-                csv,
+                normalizedCsv,
                 dryRun,
                 securityContext.getUserPrincipal().getName(),
                 recursive,

openmetadata-service/src/main/java/org/openmetadata/service/resources/columns/ColumnResource.java

@@ -40,6 +40,7 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import lombok.extern.slf4j.Slf4j;
+import org.openmetadata.csv.CsvUtil;
 import org.openmetadata.schema.api.data.BulkColumnUpdatePreview;
 import org.openmetadata.schema.api.data.BulkColumnUpdateRequest;
 import org.openmetadata.schema.api.data.ColumnGridResponse;
@@ -356,7 +357,7 @@ public Response bulkUpdateColumnsPreview(
 
   @GET
   @Path("/export")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces({"text/csv; charset=UTF-8"})
   @Operation(
       operationId = "exportUniqueColumns",
       summary = "Export unique column names to CSV",
@@ -370,7 +371,7 @@ public Response bulkUpdateColumnsPreview(
             description = "CSV export of unique columns",
             content =
                 @Content(
-                    mediaType = "text/plain",
+                    mediaType = "text/csv; charset=UTF-8",
                     schema = @Schema(implementation = String.class))),
         @ApiResponse(responseCode = "400", description = "Bad request")
       })
@@ -402,7 +403,7 @@ public String exportUniqueColumns(
 
   @POST
   @Path("/import")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Operation(
       operationId = "importUniqueColumns",
       summary = "Import column metadata from CSV (with dry-run)",
@@ -442,11 +443,12 @@ public Response importUniqueColumns(
       @Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId,
       String csv) {
 
+    String normalizedCsv = CsvUtil.stripUtf8Bom(csv);
     CsvImportResult result =
         repository.importColumnsCSV(
             uriInfo,
             securityContext,
-            csv,
+            normalizedCsv,
             dryRun,
             entityTypes,
             serviceName,
@@ -459,7 +461,7 @@ public Response importUniqueColumns(
 
   @POST
   @Path("/import-async")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Operation(
       operationId = "importUniqueColumnsAsync",
       summary = "Import column metadata from CSV asynchronously",
@@ -492,6 +494,7 @@ public Response importUniqueColumnsAsync(
       @Parameter(description = "Filter by domain ID") @QueryParam("domainId") String domainId,
       String csv) {
 
+    String normalizedCsv = CsvUtil.stripUtf8Bom(csv);
     String jobId = UUID.randomUUID().toString();
     CSVImportResponse responseEntity =
         new CSVImportResponse(jobId, "CSV column import is in progress.");
@@ -508,7 +511,7 @@ public Response importUniqueColumnsAsync(
                 repository.importColumnsCSV(
                     uriInfo,
                     securityContext,
-                    csv,
+                    normalizedCsv,
                     false,
                     entityTypes,
                     serviceName,

openmetadata-service/src/main/java/org/openmetadata/service/resources/databases/TableResource.java

@@ -582,7 +582,7 @@ public Response patch(
 
   @GET
   @Path("/name/{name}/exportAsync")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(
       operationId = "exportTable",
@@ -606,7 +606,7 @@ public Response exportCsvAsync(
 
   @GET
   @Path("/name/{name}/export")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces({"text/csv; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "exportTable",
@@ -617,7 +617,7 @@ public Response exportCsvAsync(
             description = "Exported csv with columns from the table",
             content =
                 @Content(
-                    mediaType = "application/json",
+                    mediaType = "text/csv; charset=UTF-8",
                     schema = @Schema(implementation = String.class)))
       })
   public String exportCsv(
@@ -631,7 +631,7 @@ public String exportCsv(
 
   @PUT
   @Path("/name/{name}/import")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "importTable",
@@ -665,7 +665,7 @@ public CsvImportResult importCsv(
 
   @PUT
   @Path("/name/{name}/importAsync")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "importTableAsync",

openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java

@@ -1252,7 +1252,7 @@ public Response addManyTestCasesToBundleTestSuite(
 
   @GET
   @Path("/name/{name}/export")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces({"text/csv; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "exportTestCases",
@@ -1267,7 +1267,9 @@ public Response addManyTestCasesToBundleTestSuite(
             responseCode = "200",
             description = "Exported CSV with test cases",
             content =
-                @Content(mediaType = "text/plain", schema = @Schema(implementation = String.class)))
+                @Content(
+                    mediaType = "text/csv; charset=UTF-8",
+                    schema = @Schema(implementation = String.class)))
       })
   public String exportCsv(
       @Context SecurityContext securityContext,
@@ -1315,7 +1317,7 @@ public Response exportCsvAsync(
 
   @PUT
   @Path("/name/{name}/import")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(
@@ -1360,7 +1362,7 @@ public CsvImportResult importCsv(
 
   @PUT
   @Path("/name/{name}/importAsync")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(

openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryResource.java

@@ -536,7 +536,7 @@ public String getCsvDocumentation(@Context SecurityContext securityContext) {
 
   @GET
   @Path("/name/{name}/exportAsync")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(
       operationId = "exportGlossary",
@@ -560,7 +560,7 @@ public Response exportCsvAsync(
 
   @GET
   @Path("/name/{name}/export")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "exportGlossary",
@@ -571,7 +571,7 @@ public Response exportCsvAsync(
             description = "Exported csv with glossary terms",
             content =
                 @Content(
-                    mediaType = "application/json",
+                    mediaType = "text/plain; charset=UTF-8",
                     schema = @Schema(implementation = String.class)))
       })
   public String exportCsv(
@@ -585,7 +585,7 @@ public String exportCsv(
 
   @PUT
   @Path("/name/{name}/import")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "importGlossary",
@@ -619,7 +619,7 @@ public CsvImportResult importCsv(
 
   @PUT
   @Path("/name/{name}/importAsync")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(

openmetadata-service/src/main/java/org/openmetadata/service/resources/glossary/GlossaryTermResource.java

@@ -1217,7 +1217,7 @@ public Response getTermRelationGraph(
 
   @GET
   @Path("/name/{fqn}/export")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces({"text/csv; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "exportGlossaryTerm",
@@ -1227,7 +1227,7 @@ public Response getTermRelationGraph(
         @ApiResponse(
             responseCode = "200",
             description = "Exported csv with glossary terms",
-            content = @Content(mediaType = "text/plain"))
+            content = @Content(mediaType = "text/csv; charset=UTF-8"))
       })
   public String exportCsv(
       @Context SecurityContext securityContext,
@@ -1242,7 +1242,7 @@ public String exportCsv(
 
   @GET
   @Path("/name/{fqn}/exportAsync")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(
       operationId = "exportGlossaryTermAsync",
@@ -1272,7 +1272,7 @@ public Response exportCsvAsync(
 
   @PUT
   @Path("/name/{fqn}/import")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Valid
   @Operation(
       operationId = "importGlossaryTerm",
@@ -1310,7 +1310,7 @@ public CsvImportResult importCsv(
 
   @PUT
   @Path("/name/{fqn}/importAsync")
-  @Consumes(MediaType.TEXT_PLAIN)
+  @Consumes({MediaType.TEXT_PLAIN + "; charset=UTF-8"})
   @Produces(MediaType.APPLICATION_JSON)
   @Valid
   @Operation(

openmetadata-service/src/main/java/org/openmetadata/service/resources/lineage/LineageResource.java

@@ -401,7 +401,7 @@ public Response searchDataQualityLineage(
 
   @GET
   @Path("/export")
-  @Produces(MediaType.TEXT_PLAIN)
+  @Produces({"text/csv; charset=UTF-8"})
   @Operation(
       operationId = "exportLineage",
       summary = "Export lineage",
@@ -411,8 +411,8 @@ public Response searchDataQualityLineage(
             description = "search response",
             content =
                 @Content(
-                    mediaType = "application/json",
-                    schema = @Schema(implementation = SearchResponse.class)))
+                    mediaType = "text/csv; charset=UTF-8",
+                    schema = @Schema(implementation = String.class)))
       })
   public String exportLineage(
       @Context UriInfo uriInfo,