Merge branch 'main' into aastha-live-dlq-it

Author: aasthabharill

.github/workflows/update-docs.yml

@@ -36,7 +36,7 @@ jobs:
 
       - name: Create Pull Request
         id: cpr
-        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: 'docs: update generated documentation'

.github/workflows/update-python-deps.yml

@@ -32,7 +32,7 @@ jobs:
 
       - name: Create Pull Request
         id: cpr
-        uses: peter-evans/create-pull-request@c0f553fe549906ede9cf27b5156039d195d2ece0 # v8.1.0
+        uses: peter-evans/create-pull-request@5f6978faf089d4d20b00c7766989d076bb2fc7f1 # v8.1.1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
           commit-message: 'deps: update Python dependencies'

plugins/templates-maven-plugin/pom.xml

@@ -37,7 +37,7 @@
 
   <properties>
     <maven.version>3.9.9</maven.version>
-    <dataflow.api.version>0.92.0</dataflow.api.version>
+    <dataflow.api.version>0.94.0</dataflow.api.version>
     <mojo.executor.version>2.4.1</mojo.executor.version>
     <maven-plugin.version>3.15.1</maven-plugin.version>
   </properties>

pom.xml

@@ -40,7 +40,7 @@
     <maven-checkstyle-plugin.version>3.2.1</maven-checkstyle-plugin.version>
     <maven-compiler-plugin.version>3.15.0</maven-compiler-plugin.version>
     <maven-dependency-plugin.version>3.10.0</maven-dependency-plugin.version>
-    <exec-maven-plugin.version>3.6.2</exec-maven-plugin.version>
+    <exec-maven-plugin.version>3.6.3</exec-maven-plugin.version>
     <maven-enforcer-plugin.version>3.6.2</maven-enforcer-plugin.version>
     <extra.enforcer.rules.version>1.11.0</extra.enforcer.rules.version>
     <maven-jar-plugin.version>3.5.0</maven-jar-plugin.version>
@@ -64,7 +64,7 @@
     <checkstyle.version>12.3.0</checkstyle.version>
     <commons-codec.version>1.21.0</commons-codec.version>
     <commons-compress.version>1.28.0</commons-compress.version>
-    <commons-io.version>2.18.0</commons-io.version>
+    <commons-io.version>2.21.0</commons-io.version>
     <commons-lang3.version>3.20.0</commons-lang3.version>
     <commons-configuration2.version>2.13.0</commons-configuration2.version>
     <commons-text.version>1.10.0</commons-text.version>
@@ -82,7 +82,7 @@
     <jettison.version>1.5.4</jettison.version>
     <json.version>20250517</json.version>
     <junit.version>4.13.2</junit.version>
-    <log4j-2.version>2.25.3</log4j-2.version>
+    <log4j-2.version>2.25.4</log4j-2.version>
     <mock-server-netty.version>5.15.0</mock-server-netty.version>
     <mockito.version>4.11.0</mockito.version>
     <nashorn.version>15.4</nashorn.version>
@@ -94,7 +94,7 @@
     <surefire.version>3.5.5</surefire.version>
     <testcontainers.version>1.21.4</testcontainers.version>
     <truth.version>1.4.5</truth.version>
-    <netty.version>4.1.132.Final</netty.version>
+    <netty.version>4.2.12.Final</netty.version>
     <zookeeper.version>3.8.6</zookeeper.version>
 
     <!-- Drop pinned version once maven-dependency-plugin gets past plexus-archiver 4.8.0 -->
@@ -107,7 +107,7 @@
     <postgresql.version>42.6.1</postgresql.version>
     <ojdbc8.version>23.26.1.0.0</ojdbc8.version>
     <mssql-jdbc.version>13.4.0.jre11</mssql-jdbc.version>
-    <neo4j-driver.version>4.4.18</neo4j-driver.version>
+    <neo4j-driver.version>5.28.12</neo4j-driver.version>
     <okio.version>3.16.4</okio.version>
     <jetty.version>12.1.8</jetty.version>
     <!-- Socket factory JARs for Cloud SQL -->

python/src/main/java/com/google/cloud/teleport/templates/python/BigQueryAnomalyDetection.java

@@ -63,9 +63,10 @@ public interface BigQueryAnomalyDetection {
       description = "Detector Specification (JSON)",
       helpText =
           "JSON string defining the anomaly detector. "
-              + "Example: {\"type\":\"ZScore\"} or "
-              + "{\"type\":\"ZScore\",\"config\":{\"threshold_criterion\":{\"type\":\"FixedThreshold\","
-              + "\"config\":{\"cutoff\":10}}}}")
+              + "Statistical: {\"type\":\"ZScore\"}, {\"type\":\"IQR\"}, {\"type\":\"RobustZScore\"}. "
+              + "Threshold: {\"type\":\"Threshold\",\"expression\":\"value >= 100\"}. "
+              + "RelativeChange: {\"type\":\"RelativeChange\",\"direction\":\"decrease\","
+              + "\"threshold_pct\":20,\"lookback_windows\":1}.")
   String getDetectorSpec();
 
   @TemplateParameter.Text(

python/src/main/python/bigquery-anomaly-detection/src/bqmonitor/pipeline.py

@@ -236,6 +236,8 @@
 from bqmonitor.metric import ComputeMetric
 from bqmonitor.metric import FanoutStrategy
 from bqmonitor.metric import MetricSpec
+from bqmonitor.relative_change_detector import RelativeChangeConfig
+from bqmonitor.relative_change_detector import RelativeChangeDoFn
 from bqmonitor.safe_eval import Expr
 from apache_beam.ml.anomaly.base import AnomalyPrediction
 from apache_beam.ml.anomaly.base import AnomalyResult
@@ -250,7 +252,7 @@
 
 _LOGGER = logging.getLogger(__name__)
 
-_SUPPORTED_DETECTORS = ('ZScore', 'IQR', 'RobustZScore')
+_SUPPORTED_DETECTORS = ('ZScore', 'IQR', 'RobustZScore', 'RelativeChange')
 
 
 @dataclass(frozen=True)
@@ -516,6 +518,7 @@ def process(self, element):
         {'name': 'value', 'type': 'FLOAT64', 'mode': 'REQUIRED'},
         {'name': 'score', 'type': 'FLOAT64', 'mode': 'NULLABLE'},
         {'name': 'label', 'type': 'INT64', 'mode': 'REQUIRED'},
+        {'name': 'info', 'type': 'STRING', 'mode': 'NULLABLE'},
         {'name': 'key', 'type': 'STRING', 'mode': 'NULLABLE'},
     ]
 }
@@ -535,6 +538,7 @@ def process(self, element):
         'score': float(prediction.score) if prediction.score is not None
         else None,
         'label': int(prediction.label),
+        'info': prediction.info if prediction.info else None,
     }
     if key is not None:
       row['key'] = str(key)
@@ -826,6 +830,34 @@ def _parse_detector_spec(json_str):
           "It will receive the computed metric value as 'value'.", expr_text)
     return _ThresholdAlert(expr_text)
 
+  if detector_type == 'RelativeChange':
+    config = d.get('config', {})
+    direction = d.get('direction', config.get('direction'))
+    if direction is None:
+      raise ValueError(
+          "RelativeChange detector requires 'direction' "
+          "(one of: increase, decrease, both).")
+    lookback_windows = d.get('lookback_windows',
+                             config.get('lookback_windows'))
+    if lookback_windows is None:
+      raise ValueError(
+          "RelativeChange detector requires 'lookback_windows' "
+          "(number of prior windows to compare against).")
+    threshold_pct = d.get('threshold_pct',
+                          config.get('threshold_pct'))
+    absolute_threshold = d.get('absolute_threshold',
+                               config.get('absolute_threshold'))
+    if threshold_pct is None and absolute_threshold is None:
+      raise ValueError(
+          "RelativeChange detector requires at least one of "
+          "'threshold_pct' or 'absolute_threshold'.")
+    return RelativeChangeConfig(
+        direction=direction,
+        lookback_windows=lookback_windows,
+        threshold_pct=threshold_pct,
+        absolute_threshold=absolute_threshold,
+    )
+
   if detector_type not in _SUPPORTED_DETECTORS:
     raise ValueError(
         f"Unknown detector type '{detector_type}'. "
@@ -1121,7 +1153,17 @@ def _add_offset_key(element, _wd=_window_duration, _keyed=has_group_by):
     global_metrics = (
         global_metrics | 'AddOffsetKey' >> beam.Map(_add_offset_key))
 
-  if isinstance(detector, _ThresholdAlert):
+  if isinstance(detector, RelativeChangeConfig):
+    anomalies = (
+        global_metrics
+        | 'DetectAnomalies' >> beam.ParDo(
+            RelativeChangeDoFn(
+                direction=detector.direction,
+                threshold_pct=detector.threshold_pct,
+                absolute_threshold=detector.absolute_threshold,
+                lookback_windows=detector.lookback_windows)))
+
+  elif isinstance(detector, _ThresholdAlert):
     anomalies = global_metrics | 'DetectAnomalies' >> beam.ParDo(detector)
   else:
     global_metrics = (