diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dbcdeefba..74f8ddef4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: run: chmod +x tests/install-packages.sh && ./tests/install-packages.sh - name: Build the stack - run: docker-compose up -d mysql postgres presto trino clickhouse vertica + run: docker-compose up -d mysql postgres presto trino clickhouse - name: Install Poetry run: pip install poetry @@ -62,7 +62,6 @@ jobs: DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}' # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}' DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' - DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica' DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}' MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}' run: | diff --git a/.github/workflows/ci_full.yml b/.github/workflows/ci_full.yml index efc1532a1..e48ba429f 100644 --- a/.github/workflows/ci_full.yml +++ b/.github/workflows/ci_full.yml @@ -31,7 +31,7 @@ jobs: run: chmod +x tests/install-packages.sh && ./tests/install-packages.sh - name: Build the stack - run: docker-compose up -d mysql postgres presto trino clickhouse vertica + run: docker-compose up -d mysql postgres presto trino clickhouse - name: Install Poetry run: pip install poetry @@ -58,7 +58,6 @@ jobs: DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}' DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}' DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse' - DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica' # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}' DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}' MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}' diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 95b1e7cf6..07ec26f5b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -63,6 +63,8 @@ You can create a local MySQL instance using `docker-compose up mysql`. The URI f You can also run a few servers at once. For example `docker-compose up mysql postgres presto`. Make sure to update the appropriate `TEST_*_CONN_STRING`, so that it will be included in the tests. +Vertica is no longer part of the default local Docker stack. If you need to exercise it, provide your own Vertica +instance and set `DATADIFF_VERTICA_URI` or override `TEST_VERTICA_CONN_STRING` in `tests/local_settings.py`. #### Run the tests diff --git a/README.md b/README.md index 7ea808d4d..1cf283bf6 100644 --- a/README.md +++ b/README.md @@ -176,6 +176,9 @@ Check out [documentation](https://web.archive.org/web/20240525044139/https://doc * 🟢: Implemented and thoroughly tested. * 🟡: Implemented, but not thoroughly tested yet. +Vertica support remains available in code, but the default CI and local Docker stack do not currently provision a +working Vertica instance. To test Vertica, supply your own connection via `DATADIFF_VERTICA_URI`. + Your database not listed here? - Contribute a [new database adapter](docs/new-database-driver-guide.rst) – we accept pull requests! diff --git a/data_diff/databases/duckdb.py b/data_diff/databases/duckdb.py index 9537ce50e..fb0122cad 100644 --- a/data_diff/databases/duckdb.py +++ b/data_diff/databases/duckdb.py @@ -72,6 +72,11 @@ class Dialect(BaseDialect): def quote(self, s: str): return f'"{s}"' + def type_repr(self, t) -> str: + if isinstance(t, TimestampTZ): + return "TIMESTAMP WITH TIME ZONE" + return super().type_repr(t) + def to_string(self, s: str): return f"{s}::VARCHAR" @@ -165,17 +170,11 @@ def create_connection(self): def select_table_schema(self, path: DbPath) -> str: database, schema, table = self._normalize_table_path(path) - - info_schema_path = ["information_schema", "columns"] - - if database: - info_schema_path.insert(0, database) - dynamic_database_clause = f"'{database}'" - else: - dynamic_database_clause = "current_catalog()" + dynamic_database_clause = f"'{database}'" if database else "current_catalog()" return ( - f"SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM {'.'.join(info_schema_path)} " + "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale " + "FROM information_schema.columns " f"WHERE table_name = '{table}' AND table_schema = '{schema}' and table_catalog = {dynamic_database_clause}" ) diff --git a/dev/Dockerfile.prestosql.340 b/dev/Dockerfile.prestosql.340 index 0b2111374..1c99b3707 100644 --- a/dev/Dockerfile.prestosql.340 +++ b/dev/Dockerfile.prestosql.340 @@ -1,4 +1,4 @@ -FROM openjdk:11-jdk-slim-buster +FROM eclipse-temurin:11-jdk-jammy ENV PRESTO_VERSION=340 ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/io/prestosql/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz @@ -10,7 +10,7 @@ WORKDIR $PRESTO_HOME RUN set -xe \ && apt-get update \ - && apt-get install -y curl less python \ + && apt-get install -y curl less python-is-python3 \ && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \ && curl -sSL $PRESTO_CLI_URL > ./bin/presto \ && chmod +x ./bin/presto \ diff --git a/dev/dev.env b/dev/dev.env index cb5eea12c..7643967e8 100644 --- a/dev/dev.env +++ b/dev/dev.env @@ -11,13 +11,3 @@ CLICKHOUSE_USER=clickhouse CLICKHOUSE_PASSWORD=Password1 CLICKHOUSE_DB=clickhouse CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1 - -# Vertica credentials -APP_DB_USER=vertica -APP_DB_PASSWORD=Password1 -VERTICA_DB_NAME=vertica - -# To prevent generating sample demo VMart data (more about it here https://www.vertica.com/docs/9.2.x/HTML/Content/Authoring/GettingStartedGuide/IntroducingVMart/IntroducingVMart.htm), -# leave VMART_DIR and VMART_ETL_SCRIPT empty. -VMART_DIR= -VMART_ETL_SCRIPT= diff --git a/docker-compose.yml b/docker-compose.yml index d2087513a..c3daba3ae 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,3 @@ -version: "3.8" - services: postgres: container_name: dd-postgresql @@ -99,31 +97,10 @@ services: networks: - local - vertica: - container_name: dd-vertica - image: vertica/vertica-ce:12.0.0-0 - restart: always - volumes: - - vertica-data:/data:delegated - ports: - - '5433:5433' - - '5444:5444' - expose: - - '5433' - - '5444' - env_file: - - dev/dev.env - tty: true - networks: - - local - - - volumes: postgresql-data: mysql-data: clickhouse-data: - vertica-data: networks: local: diff --git a/tests/common.py b/tests/common.py index 33e39b21c..405c4b440 100644 --- a/tests/common.py +++ b/tests/common.py @@ -37,10 +37,8 @@ TEST_CLICKHOUSE_CONN_STRING: str = ( os.environ.get("DATADIFF_CLICKHOUSE_URI", "clickhouse://clickhouse:Password1@localhost:9000/clickhouse") or None ) -# vertica uri provided for docker - "vertica://vertica:Password1@localhost:5433/vertica" -TEST_VERTICA_CONN_STRING: str = ( - os.environ.get("DATADIFF_VERTICA_URI", "vertica://vertica:Password1@localhost:5433/vertica") or None -) +# Vertica is opt-in: the repo no longer provides a working local default service. +TEST_VERTICA_CONN_STRING: str = os.environ.get("DATADIFF_VERTICA_URI") or None TEST_DUCKDB_CONN_STRING: str = os.environ.get("DATADIFF_DUCKDB_URI", "duckdb://main:@:memory:") or None TEST_MSSQL_CONN_STRING: str = os.environ.get("DATADIFF_MSSQL_URI") diff --git a/tests/test_duckdb.py b/tests/test_duckdb.py index 10c3644e3..9e9868ddc 100644 --- a/tests/test_duckdb.py +++ b/tests/test_duckdb.py @@ -38,5 +38,5 @@ def test_select_table_schema(self): self.assertEqual(self.duckdb_conn.select_table_schema(db_path), expected_sql) db_path = ("custom_db", "custom_schema", "test_table") - expected_sql = "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM custom_db.information_schema.columns WHERE table_name = 'test_table' AND table_schema = 'custom_schema' and table_catalog = 'custom_db'" + expected_sql = "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM information_schema.columns WHERE table_name = 'test_table' AND table_schema = 'custom_schema' and table_catalog = 'custom_db'" self.assertEqual(self.duckdb_conn.select_table_schema(db_path), expected_sql) diff --git a/tests/waiting_for_stack_up.sh b/tests/waiting_for_stack_up.sh index 762138ded..c4f96113d 100755 --- a/tests/waiting_for_stack_up.sh +++ b/tests/waiting_for_stack_up.sh @@ -1,6 +1,6 @@ #!/bin/bash -if [ -n "$DATADIFF_VERTICA_URI" ] +if [ -n "${DATADIFF_VERTICA_URI:-}" ] then echo "Check Vertica DB running..." while true