Skip to content

Commit 3023a29

Browse files
SaaiAravindhRajaSaaiAravindhRaja
authored
Merge branch 'main' into feat/issue-12787-hive-metastore-mssql-oracle
2 parents 6d77886 + 2ce8fb0 commit 3023a29

File tree

77 files changed

+4900
-272
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

77 files changed

+4900
-272
lines changed

‎.github/workflows/playwright-sso-login-nightly.yml‎

Lines changed: 145 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,145 @@
1+
# Copyright 2025 Collate
2+
# Licensed under the Apache License, Version 2.0 (the "License");
3+
# you may not use this file except in compliance with the License.
4+
# You may obtain a copy of the License at
5+
# http://www.apache.org/licenses/LICENSE-2.0
6+
# Unless required by applicable law or agreed to in writing, software
7+
# distributed under the License is distributed on an "AS IS" BASIS,
8+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
9+
# See the License for the specific language governing permissions and
10+
# limitations under the License.
11+
12+
name: SSO Login Nightly
13+
14+
on:
15+
schedule:
16+
- cron: '0 3 * * *'
17+
workflow_dispatch:
18+
inputs:
19+
sso_provider:
20+
description: 'SSO provider (or "all")'
21+
required: true
22+
default: okta
23+
type: choice
24+
options:
25+
- okta
26+
- keycloak-azure-saml
27+
- all
28+
29+
permissions:
30+
contents: read
31+
32+
concurrency:
33+
group: sso-login-nightly-${{ github.event.inputs.sso_provider || 'scheduled' }}
34+
cancel-in-progress: true
35+
36+
jobs:
37+
# To onboard a new provider:
38+
# 1. Add a matrix entry below (`name` is the lowercase provider id used by
39+
# the Playwright helper; `env_prefix` is the uppercase/underscore form
40+
# used to look up credentials). Also add `name` to the dispatch
41+
# `options:` list above.
42+
# 2. Add <ENV_PREFIX>_SSO_USERNAME (variable) and <ENV_PREFIX>_SSO_PASSWORD
43+
# (variable) to the `test` environment. Use a secret instead of a
44+
# variable for the password if the provider uses a real (non-fixture)
45+
# credential.
46+
# 3. Register the helper in playwright/utils/sso-providers/index.ts.
47+
sso-login:
48+
runs-on: ubuntu-latest
49+
environment: test
50+
timeout-minutes: 45
51+
strategy:
52+
fail-fast: false
53+
matrix:
54+
provider:
55+
${{ (github.event_name == 'schedule' || github.event.inputs.sso_provider == 'all')
56+
&& fromJSON('[{"name":"okta","env_prefix":"OKTA"},{"name":"keycloak-azure-saml","env_prefix":"KEYCLOAK_AZURE_SAML"}]')
57+
|| (github.event.inputs.sso_provider == 'keycloak-azure-saml'
58+
&& fromJSON('[{"name":"keycloak-azure-saml","env_prefix":"KEYCLOAK_AZURE_SAML"}]')
59+
|| fromJSON('[{"name":"okta","env_prefix":"OKTA"}]')) }}
60+
steps:
61+
- name: Free Disk Space (Ubuntu)
62+
uses: jlumbroso/free-disk-space@main
63+
with:
64+
tool-cache: false
65+
android: true
66+
dotnet: true
67+
haskell: true
68+
large-packages: false
69+
swap-storage: true
70+
docker-images: false
71+
72+
- name: Checkout
73+
uses: actions/checkout@v4
74+
75+
- name: Cache Maven Dependencies
76+
uses: actions/cache@v4
77+
with:
78+
path: ~/.m2
79+
key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
80+
restore-keys: |
81+
${{ runner.os }}-maven-
82+
83+
- name: Setup OpenMetadata Test Environment
84+
uses: ./.github/actions/setup-openmetadata-test-environment
85+
with:
86+
python-version: '3.10'
87+
args: '-d postgresql -i false'
88+
ingestion_dependency: 'all'
89+
90+
- name: Setup Node.js
91+
uses: actions/setup-node@v4
92+
with:
93+
node-version-file: 'openmetadata-ui/src/main/resources/ui/.nvmrc'
94+
95+
- name: Install dependencies
96+
working-directory: openmetadata-ui/src/main/resources/ui/
97+
run: yarn --ignore-scripts --frozen-lockfile
98+
99+
- name: Install Playwright Browsers
100+
run: npx playwright@1.57.0 install chromium --with-deps
101+
102+
- name: Start Keycloak SAML IdP
103+
if: startsWith(matrix.provider.name, 'keycloak-')
104+
run: |
105+
docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml up -d
106+
timeout 180 bash -c 'until curl -fsS http://localhost:8080/realms/om-azure-saml >/dev/null; do sleep 2; done'
107+
108+
- name: Run SSO Login Spec
109+
working-directory: openmetadata-ui/src/main/resources/ui
110+
env:
111+
SSO_PROVIDER_TYPE: ${{ matrix.provider.name }}
112+
SSO_USERNAME: ${{ vars[format('{0}_SSO_USERNAME', matrix.provider.env_prefix)] }}
113+
SSO_PASSWORD: ${{ vars[format('{0}_SSO_PASSWORD', matrix.provider.env_prefix)] || secrets[format('{0}_SSO_PASSWORD', matrix.provider.env_prefix)] }}
114+
KEYCLOAK_SAML_BASE_URL: http://localhost:8080
115+
PLAYWRIGHT_IS_OSS: true
116+
run: |
117+
npx playwright test playwright/e2e/Auth/SSOLogin.spec.ts \
118+
--project=sso-auth \
119+
--workers=1
120+
121+
- name: Upload HTML report
122+
if: always()
123+
uses: actions/upload-artifact@v4
124+
with:
125+
name: sso-login-html-report-${{ matrix.provider.name }}
126+
path: openmetadata-ui/src/main/resources/ui/playwright/output/playwright-report
127+
retention-days: 5
128+
129+
- name: Send Slack Notification
130+
if: always()
131+
working-directory: openmetadata-ui/src/main/resources/ui
132+
env:
133+
RUN_TITLE: "SSO Login Nightly: ${{ matrix.provider.name }} (${{ github.ref_name }})"
134+
RUN_URL: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
135+
SLACK_BOT_USER_OAUTH_TOKEN: ${{ secrets.E2E_SLACK_BOT_OAUTH_TOKEN }}
136+
run: |
137+
npx playwright-slack-report -c playwright/slack-cli.config.json -j playwright/output/results.json > slack_report.json
138+
139+
- name: Clean Up
140+
if: always()
141+
run: |
142+
docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml down --remove-orphans || true
143+
cd ./docker/development
144+
docker compose down --remove-orphans
145+
sudo rm -rf ${PWD}/docker-volume

‎bootstrap/sql/migrations/native/1.12.0/mysql/postDataMigrationSQLScript.sql‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,4 +76,3 @@ SET json = JSON_REMOVE(JSON_REMOVE(json, '$.inputPorts'), '$.outputPorts')
7676
WHERE jsonSchema = 'dataProduct'
7777
AND (JSON_CONTAINS_PATH(json, 'one', '$.inputPorts')
7878
OR JSON_CONTAINS_PATH(json, 'one', '$.outputPorts'));
79-

‎bootstrap/sql/migrations/native/1.12.0/postgres/postDataMigrationSQLScript.sql‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -110,3 +110,4 @@ SET json = json::jsonb - 'inputPorts' - 'outputPorts'
110110
WHERE jsonSchema = 'dataProduct'
111111
AND (json::jsonb ?? 'inputPorts' OR json::jsonb ?? 'outputPorts');
112112

113+

‎bootstrap/sql/migrations/native/1.12.6/mysql/postDataMigrationSQLScript.sql‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,27 @@
1-
-- Placeholder for 1.12.6 MySQL post data migration SQL script
1+
-- Remove pipeline annotation from service-level, domain-level, and dataProduct-level lineage edges.
2+
-- These edges incorrectly inherited the pipeline annotation from entity-level lineage, causing service
3+
-- nodes to appear in entity-level lineage views and the "By Service" view to be empty for pipeline
4+
-- entities. After this migration, run an Elasticsearch/OpenSearch reindex to update search documents.
5+
UPDATE entity_relationship
6+
SET json = JSON_REMOVE(json, '$.pipeline')
7+
WHERE fromEntity IN ('databaseService', 'messagingService', 'pipelineService', 'dashboardService',
8+
'mlmodelService', 'metadataService', 'storageService', 'searchService', 'apiService',
9+
'driveService')
10+
AND toEntity IN ('databaseService', 'messagingService', 'pipelineService', 'dashboardService',
11+
'mlmodelService', 'metadataService', 'storageService', 'searchService', 'apiService',
12+
'driveService')
13+
AND relation = 13
14+
AND JSON_CONTAINS_PATH(json, 'one', '$.pipeline');
15+
16+
UPDATE entity_relationship
17+
SET json = JSON_REMOVE(json, '$.pipeline')
18+
WHERE fromEntity = 'domain' AND toEntity = 'domain'
19+
AND relation = 13
20+
AND JSON_EXTRACT(json, '$.pipeline') IS NOT NULL;
21+
22+
UPDATE entity_relationship
23+
SET json = JSON_REMOVE(json, '$.pipeline')
24+
WHERE fromEntity = 'dataProduct' AND toEntity = 'dataProduct'
25+
AND relation = 13
26+
AND JSON_EXTRACT(json, '$.pipeline') IS NOT NULL;
27+

‎bootstrap/sql/migrations/native/1.12.6/postgres/postDataMigrationSQLScript.sql‎

Lines changed: 27 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1,27 @@
1-
-- Placeholder for 1.12.6 Postgres post data migration SQL script
1+
-- Remove pipeline annotation from service-level, domain-level, and dataProduct-level lineage edges.
2+
-- These edges incorrectly inherited the pipeline annotation from entity-level lineage, causing service
3+
-- nodes to appear in entity-level lineage views and the "By Service" view to be empty for pipeline
4+
-- entities. After this migration, run an Elasticsearch/OpenSearch reindex to update search documents.
5+
UPDATE entity_relationship
6+
SET json = json - 'pipeline'
7+
WHERE fromentity IN ('databaseService', 'messagingService', 'pipelineService', 'dashboardService',
8+
'mlmodelService', 'metadataService', 'storageService', 'searchService', 'apiService',
9+
'driveService')
10+
AND toentity IN ('databaseService', 'messagingService', 'pipelineService', 'dashboardService',
11+
'mlmodelService', 'metadataService', 'storageService', 'searchService', 'apiService',
12+
'driveService')
13+
AND relation = 13
14+
AND json ?? 'pipeline';
15+
16+
UPDATE entity_relationship
17+
SET json = json - 'pipeline'
18+
WHERE fromentity = 'domain' AND toentity = 'domain'
19+
AND relation = 13
20+
AND json ?? 'pipeline';
21+
22+
UPDATE entity_relationship
23+
SET json = json - 'pipeline'
24+
WHERE fromentity = 'dataProduct' AND toentity = 'dataProduct'
25+
AND relation = 13
26+
AND json ?? 'pipeline';
27+

‎bootstrap/sql/migrations/native/1.12.6/postgres/schemaChanges.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
-- Placeholder for 1.12.6 Postgres schema changes
1+
-- Placeholder for 1.12.6 PostgreSQL schema changes

‎conf/openmetadata.yaml‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,16 @@ qos:
162162
maxSuspendedRequestCount: ${QOS_MAX_SUSPENDED_REQUEST_COUNT:-1000}
163163
maxSuspendSeconds: ${QOS_MAX_SUSPEND_SECONDS:-30}
164164

165+
cacheMemory:
166+
# Entity JSON caches (CACHE_WITH_ID, CACHE_WITH_NAME) — weight-based eviction.
167+
# Entity JSON can range from 1KB to 2MB+. Increase on high-memory deployments for better hit rates.
168+
entityCacheMaxSizeBytes: ${ENTITY_CACHE_MAX_SIZE_BYTES:-104857600} # 100 MB
169+
entityCacheTTLSeconds: ${ENTITY_CACHE_TTL_SECONDS:-30}
170+
# Auth caches (user context + policies) — TTLs hardcoded (2min policies, 15min user context)
171+
authCacheMaxEntries: ${AUTH_CACHE_MAX_ENTRIES:-5000}
172+
# RBAC query cache (OpenSearch role-based access control query DSL)
173+
rbacCacheMaxEntries: ${RBAC_CACHE_MAX_ENTRIES:-5000}
174+
165175
# Logging settings.
166176
# https://logback.qos.ch/manual/layouts.html#conversionWord
167177
# Set LOG_FORMAT=json for structured logs. The default text format preserves legacy output.

‎docker/docker-compose-openmetadata/env-mysql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ SMTP_SERVER_STRATEGY="SMTP_TLS"
143143
OM_RESOURCE_PACKAGES="[]"
144144
OM_EXTENSIONS="[]"
145145
# Heap OPTS Configurations
146-
OPENMETADATA_HEAP_OPTS="-Xmx1G -Xms1G"
146+
OPENMETADATA_HEAP_OPTS="-Xmx2G -Xms256M -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError"
147147
# Application Config
148148
CUSTOM_LOGO_URL_PATH=""
149149
CUSTOM_MONOGRAM_URL_PATH=""

‎docker/docker-compose-openmetadata/env-postgres‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ SMTP_SERVER_STRATEGY="SMTP_TLS"
143143
OM_RESOURCE_PACKAGES="[]"
144144
OM_EXTENSIONS="[]"
145145
# Heap OPTS Configurations
146-
OPENMETADATA_HEAP_OPTS="-Xmx1G -Xms1G"
146+
OPENMETADATA_HEAP_OPTS="-Xmx2G -Xms512M -XX:+UseG1GC -XX:MaxGCPauseMillis=200 -XX:+HeapDumpOnOutOfMemoryError"
147147
# Application Config
148148
CUSTOM_LOGO_URL_PATH=""
149149
CUSTOM_MONOGRAM_URL_PATH=""

‎docker/local-sso/keycloak-saml/README.md‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Keycloak SAML Fixture
2+
3+
Local SAML IdP fixture for the Playwright SSO login spec.
4+
5+
```bash
6+
docker compose -f docker/local-sso/keycloak-saml/docker-compose.yml up -d
7+
```
8+
9+
It imports one realm for an OpenMetadata server running at `http://localhost:8585`:
10+
11+
- `om-azure-saml`
12+
- User: `azure.saml@openmetadata.local`
13+
- Password: `OpenMetadata@123`
14+
15+
Use the matching Playwright provider type:
16+
17+
```bash
18+
SSO_PROVIDER_TYPE=keycloak-azure-saml \
19+
SSO_USERNAME=azure.saml@openmetadata.local \
20+
SSO_PASSWORD=OpenMetadata@123 \
21+
npx playwright test playwright/e2e/Auth/SSOLogin.spec.ts --project=sso-auth --workers=1
22+
```

0 commit comments

Comments
 (0)