Fix test harness - update docker images

.github/workflows/ci.yml

@@ -30,7 +30,7 @@ jobs:
         run:  chmod +x tests/install-packages.sh && ./tests/install-packages.sh
 
       - name: Build the stack
-        run: docker-compose up -d mysql postgres presto trino clickhouse vertica
+        run: docker-compose up -d mysql postgres presto trino clickhouse
 
       - name: Install Poetry
         run: pip install poetry
@@ -62,7 +62,6 @@ jobs:
             DATADIFF_TRINO_URI: '${{ secrets.DATADIFF_TRINO_URI }}'
             # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
-            DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
             DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}'
             MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}'
         run: |

.github/workflows/ci_full.yml

@@ -31,7 +31,7 @@ jobs:
         run:  chmod +x tests/install-packages.sh && ./tests/install-packages.sh
 
       - name: Build the stack
-        run: docker-compose up -d mysql postgres presto trino clickhouse vertica
+        run: docker-compose up -d mysql postgres presto trino clickhouse
 
       - name: Install Poetry
         run: pip install poetry
@@ -58,7 +58,6 @@ jobs:
             DATADIFF_SNOWFLAKE_URI: '${{ secrets.DATADIFF_SNOWFLAKE_URI }}'
             DATADIFF_PRESTO_URI: '${{ secrets.DATADIFF_PRESTO_URI }}'
             DATADIFF_CLICKHOUSE_URI: 'clickhouse://clickhouse:Password1@localhost:9000/clickhouse'
-            DATADIFF_VERTICA_URI: 'vertica://vertica:Password1@localhost:5433/vertica'
             # DATADIFF_BIGQUERY_URI: '${{ secrets.DATADIFF_BIGQUERY_URI }}'
             DATADIFF_REDSHIFT_URI: '${{ secrets.DATADIFF_REDSHIFT_URI }}'
             MOTHERDUCK_TOKEN: '${{ secrets.MOTHERDUCK_TOKEN }}'

CONTRIBUTING.md

@@ -63,6 +63,8 @@ You can create a local MySQL instance using `docker-compose up mysql`. The URI f
 You can also run a few servers at once. For example `docker-compose up mysql postgres presto`.
 
 Make sure to update the appropriate `TEST_*_CONN_STRING`, so that it will be included in the tests.
+Vertica is no longer part of the default local Docker stack. If you need to exercise it, provide your own Vertica
+instance and set `DATADIFF_VERTICA_URI` or override `TEST_VERTICA_CONN_STRING` in `tests/local_settings.py`.
 
 #### Run the tests
 

README.md

@@ -176,6 +176,9 @@ Check out [documentation](https://web.archive.org/web/20240525044139/https://doc
 * 🟢: Implemented and thoroughly tested.
 * 🟡: Implemented, but not thoroughly tested yet.
 
+Vertica support remains available in code, but the default CI and local Docker stack do not currently provision a
+working Vertica instance. To test Vertica, supply your own connection via `DATADIFF_VERTICA_URI`.
+
 Your database not listed here?
 
 - Contribute a [new database adapter](docs/new-database-driver-guide.rst) – we accept pull requests!

data_diff/databases/duckdb.py

@@ -72,6 +72,11 @@ class Dialect(BaseDialect):
     def quote(self, s: str):
         return f'"{s}"'
 
+    def type_repr(self, t) -> str:
+        if isinstance(t, TimestampTZ):
+            return "TIMESTAMP WITH TIME ZONE"
+        return super().type_repr(t)
+
     def to_string(self, s: str):
         return f"{s}::VARCHAR"
 
@@ -165,17 +170,11 @@ def create_connection(self):
 
     def select_table_schema(self, path: DbPath) -> str:
         database, schema, table = self._normalize_table_path(path)
-
-        info_schema_path = ["information_schema", "columns"]
-
-        if database:
-            info_schema_path.insert(0, database)
-            dynamic_database_clause = f"'{database}'"
-        else:
-            dynamic_database_clause = "current_catalog()"
+        dynamic_database_clause = f"'{database}'" if database else "current_catalog()"
 
         return (
-            f"SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM {'.'.join(info_schema_path)} "
+            "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale "
+            "FROM information_schema.columns "
             f"WHERE table_name = '{table}' AND table_schema = '{schema}' and table_catalog = {dynamic_database_clause}"
         )
 

dev/Dockerfile.prestosql.340

@@ -1,4 +1,4 @@
-FROM openjdk:11-jdk-slim-buster
+FROM eclipse-temurin:11-jdk-jammy
 
 ENV PRESTO_VERSION=340
 ENV PRESTO_SERVER_URL=https://repo1.maven.org/maven2/io/prestosql/presto-server/${PRESTO_VERSION}/presto-server-${PRESTO_VERSION}.tar.gz
@@ -10,7 +10,7 @@ WORKDIR $PRESTO_HOME
 
 RUN set -xe \
     && apt-get update \
-    && apt-get install -y curl less python \
+    && apt-get install -y curl less python-is-python3 \
     && curl -sSL $PRESTO_SERVER_URL | tar xz --strip 1 \
     && curl -sSL $PRESTO_CLI_URL > ./bin/presto \
     && chmod +x ./bin/presto \

dev/dev.env

@@ -11,13 +11,3 @@ CLICKHOUSE_USER=clickhouse
 CLICKHOUSE_PASSWORD=Password1
 CLICKHOUSE_DB=clickhouse
 CLICKHOUSE_DEFAULT_ACCESS_MANAGEMENT=1
-
-# Vertica credentials
-APP_DB_USER=vertica
-APP_DB_PASSWORD=Password1
-VERTICA_DB_NAME=vertica
-
-# To prevent generating sample demo VMart data (more about it here https://www.vertica.com/docs/9.2.x/HTML/Content/Authoring/GettingStartedGuide/IntroducingVMart/IntroducingVMart.htm),
-# leave VMART_DIR and VMART_ETL_SCRIPT empty.
-VMART_DIR=
-VMART_ETL_SCRIPT=

docker-compose.yml

@@ -1,5 +1,3 @@
-version: "3.8"
-
 services:
     postgres:
       container_name: dd-postgresql
@@ -99,31 +97,10 @@ services:
         networks:
             - local
 
-    vertica:
-      container_name: dd-vertica
-      image: vertica/vertica-ce:12.0.0-0
-      restart: always
-      volumes:
-          - vertica-data:/data:delegated
-      ports:
-        - '5433:5433'
-        - '5444:5444'
-      expose:
-        - '5433'
-        - '5444'
-      env_file:
-        - dev/dev.env
-      tty: true
-      networks:
-      - local
-
-
-
 volumes:
   postgresql-data:
   mysql-data:
   clickhouse-data:
-  vertica-data:
 
 networks:
   local:

tests/common.py

@@ -37,10 +37,8 @@
 TEST_CLICKHOUSE_CONN_STRING: str = (
     os.environ.get("DATADIFF_CLICKHOUSE_URI", "clickhouse://clickhouse:Password1@localhost:9000/clickhouse") or None
 )
-# vertica uri provided for docker - "vertica://vertica:Password1@localhost:5433/vertica"
-TEST_VERTICA_CONN_STRING: str = (
-    os.environ.get("DATADIFF_VERTICA_URI", "vertica://vertica:Password1@localhost:5433/vertica") or None
-)
+# Vertica is opt-in: the repo no longer provides a working local default service.
+TEST_VERTICA_CONN_STRING: str = os.environ.get("DATADIFF_VERTICA_URI") or None
 TEST_DUCKDB_CONN_STRING: str = os.environ.get("DATADIFF_DUCKDB_URI", "duckdb://main:@:memory:") or None
 TEST_MSSQL_CONN_STRING: str = os.environ.get("DATADIFF_MSSQL_URI")
 

tests/test_duckdb.py

@@ -38,5 +38,5 @@ def test_select_table_schema(self):
         self.assertEqual(self.duckdb_conn.select_table_schema(db_path), expected_sql)
 
         db_path = ("custom_db", "custom_schema", "test_table")
-        expected_sql = "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM custom_db.information_schema.columns WHERE table_name = 'test_table' AND table_schema = 'custom_schema' and table_catalog = 'custom_db'"
+        expected_sql = "SELECT column_name, data_type, datetime_precision, numeric_precision, numeric_scale FROM information_schema.columns WHERE table_name = 'test_table' AND table_schema = 'custom_schema' and table_catalog = 'custom_db'"
         self.assertEqual(self.duckdb_conn.select_table_schema(db_path), expected_sql)

tests/waiting_for_stack_up.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-if [ -n "$DATADIFF_VERTICA_URI" ]
+if [ -n "${DATADIFF_VERTICA_URI:-}" ]
     then
         echo "Check Vertica DB running..."
         while true