ci: add trend tracking and split health report into package

Add per-job trend tracking: failure rates are bucketed into natural time
units (daily/weekly/monthly) and compared first-half vs second-half to
produce a directional arrow and Δ% (e.g. ↑ +45.0%, ↓ -40.0%). Jobs with
fewer than num_buckets * 2 runs show — to avoid misleading results from
sparse data.

Move the script and its tests into .github/scripts/ci_health_report/ as
separate files (ci_health_report.py / test_ci_health_report.py), and add
simulate_report.py for local testing with synthetic data.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Signed-off-by: John Ramsden <john.ramsden@canonical.com>

Author: johnramsden

.github/scripts/ci_health_report.py …pts/ci_health_report/ci_health_report.py.github/scripts/ci_health_report.py renamed to .github/scripts/ci_health_report/ci_health_report.py .github/scripts/ci_health_report.py renamed to .github/scripts/ci_health_report/ci_health_report.py

@@ -24,6 +24,38 @@
 COUNTED_CONCLUSIONS = {"success", "failure"}
 
 
+def bucket_count(lookback_days):
+    """Return the number of trend buckets for a given lookback window.
+
+    Uses natural time units so bucket boundaries are semantically meaningful:
+      - daily  for windows up to 14 days
+      - weekly for windows up to 90 days
+      - ~monthly (28-day) for longer windows
+    """
+    if lookback_days <= 14:
+        return lookback_days
+    elif lookback_days <= 90:
+        return lookback_days // 7
+    else:
+        return lookback_days // 28
+
+
+def trend_indicator(buckets):
+    """Compare first-half vs second-half failure rate and return an arrow + delta string."""
+    mid = len(buckets) // 2
+    early, recent = buckets[:mid], buckets[mid:]
+    e_runs  = sum(b["runs"]     for b in early)
+    e_fails = sum(b["failures"] for b in early)
+    r_runs  = sum(b["runs"]     for b in recent)
+    r_fails = sum(b["failures"] for b in recent)
+    if e_runs == 0 or r_runs == 0:
+        return "—"
+    delta = (r_fails / r_runs - e_fails / e_runs) * 100
+    if abs(delta) < 1.0:
+        return f"→ {delta:+.1f}%"
+    return f"{'↑' if delta > 0 else '↓'} {delta:+.1f}%"
+
+
 def _headers(token):
     return {
         "Authorization": f"Bearer {token}",
@@ -45,7 +77,7 @@ def _urlopen(req):
         if retry_after:
             wait = int(retry_after) + 5
         elif reset:
-            wait = max(0, int(reset) - int(time.time())) + 5
+            wait = max(0, int(reset) - int(time.time())) + 5 # Seconds until reset + 5
         else:
             wait = 60
         print(f"Rate limited (HTTP {e.code}). Waiting {wait}s before retry...", file=sys.stderr)
@@ -122,7 +154,9 @@ def build_report(stats, lookback_days, top_n, now):
     table_lines = []
     for (workflow, job), s in rows:
         rate = s["failures"] / s["runs"] * 100
-        table_lines.append(f"| {workflow} | {job} | {s['runs']} | {s['failures']} | {rate:.1f}% |")
+        min_runs = len(s["buckets"]) * 2
+        trend = trend_indicator(s["buckets"]) if s["runs"] >= min_runs else "—"
+        table_lines.append(f"| {workflow} | {job} | {s['runs']} | {s['failures']} | {rate:.1f}% | {trend} |")
 
     # Top N by absolute failure count
     top = sorted(stats.items(), key=lambda x: x[1]["failures"], reverse=True)[:top_n]
@@ -144,10 +178,16 @@ def build_report(stats, lookback_days, top_n, now):
         "",
         "### Job Failure Rates",
         "",
-        "| Workflow | Job | Runs | Failures | Rate |",
-        "|----------|-----|------|----------|------|",
+        "| Workflow | Job | Runs | Failures | Rate | Trend |",
+        "|----------|-----|------|----------|------|-------|",
         *table_lines,
         "",
+        f"_Trend: the {lookback_days}-day window is divided into equal time buckets"
+        " (daily for ≤ 14 days, weekly for ≤ 90 days, ~monthly beyond that)."
+        " The failure rate in the first half of those buckets is compared to the second half:"
+        " ↑ = getting worse, ↓ = improving, → = stable (< 1 pp change)."
+        " — = fewer than 2 runs per bucket on average; not enough data._",
+        "",
         f"### Top {top_n} Most Failing Jobs",
         "",
         *top_lines,
@@ -176,7 +216,9 @@ def main():
     top_jobs = int(top_jobs_str)
 
     now = datetime.now(timezone.utc)
-    since = (now - timedelta(days=lookback_days)).strftime("%Y-%m-%dT%H:%M:%SZ")
+    since_dt = now - timedelta(days=lookback_days)
+    since = since_dt.strftime("%Y-%m-%dT%H:%M:%SZ")
+    num_buckets = bucket_count(lookback_days)
 
     print(f"Fetching workflow runs since {since}...")
     runs = get_runs(token, repo, since)
@@ -186,21 +228,35 @@ def main():
         print("No runs found. Skipping report.")
         return
 
-    # Aggregate: (workflow_name, job_name) -> {runs, failures}
+    # Aggregate: (workflow_name, job_name) -> {runs, failures, buckets}
     # Only "success" and "failure" conclusions are counted; skipped/cancelled are excluded.
-    stats = defaultdict(lambda: {"runs": 0, "failures": 0})
+    # Buckets divide the lookback window into equal time slices (oldest → newest) for trend tracking.
+    stats = defaultdict(lambda: {
+        "runs": 0,
+        "failures": 0,
+        "buckets": [{"runs": 0, "failures": 0} for _ in range(num_buckets)],
+    })
+    window_secs = (now - since_dt).total_seconds()
 
     for i, run in enumerate(runs, start=1):
         print(f"  Fetching jobs for run {i}/{len(runs)} (id={run['id']})...")
+        run_dt = datetime.fromisoformat(run["created_at"].replace("Z", "+00:00"))
+        elapsed = (run_dt - since_dt).total_seconds()  # seconds from window start to this run
+        # clamp: elapsed==window_secs would produce index num_buckets
+        bucket_idx = min(int(elapsed / window_secs * num_buckets), num_buckets - 1)
+        bucket_idx = max(0, bucket_idx)  # clamp: clock skew can make elapsed slightly negative
+
         jobs = get_jobs(token, repo, run["id"])
         for job in jobs:
             conclusion = job.get("conclusion")
             if conclusion not in COUNTED_CONCLUSIONS:
                 continue
             key = (run["name"], job["name"])
             stats[key]["runs"] += 1
+            stats[key]["buckets"][bucket_idx]["runs"] += 1
             if conclusion == "failure":
                 stats[key]["failures"] += 1
+                stats[key]["buckets"][bucket_idx]["failures"] += 1
 
     if not stats:
         print("No job data collected. Skipping report.")
@@ -218,63 +274,6 @@ def main():
     post_comment(token, repo, issue_number, report)
     print("Report generated successfully.")
 
-## TESTS ##
-
-import unittest
-from unittest.mock import MagicMock, patch
-
-
-class _Tests(unittest.TestCase):
-
-    @patch("time.sleep")
-    @patch("urllib.request.urlopen")
-    def test_rate_limit_retries_with_wait(self, mock_urlopen, mock_sleep):
-        """_urlopen sleeps Retry-After + 5s on 429 then retries successfully."""
-        import http.client
-        msg = http.client.HTTPMessage()
-        msg["Retry-After"] = "10"
-        resp = MagicMock()
-        resp.read.return_value = b'{"ok": true}'
-        resp.__enter__ = lambda s: s
-        resp.__exit__ = MagicMock(return_value=False)
-        mock_urlopen.side_effect = [
-            urllib.error.HTTPError("https://api.github.com/test", 429, "Too Many Requests", msg, None),
-            resp,
-        ]
-        result = _urlopen(urllib.request.Request("https://api.github.com/test"))
-        mock_sleep.assert_called_once_with(15)  # Retry-After(10) + 5
-        self.assertEqual(result, b'{"ok": true}')
-
-    def test_build_report_structure_and_totals(self):
-        """build_report produces a markdown table and correct summary totals."""
-        stats = defaultdict(lambda: {"runs": 0, "failures": 0})
-        stats[("Tests", "build")]["runs"] = 10
-        stats[("Tests", "build")]["failures"] = 3
-        now = datetime(2026, 1, 1, 9, 0, 0, tzinfo=timezone.utc)
-        report = build_report(stats, 30, 5, now)
-        self.assertIn("| Workflow | Job | Runs | Failures | Rate |", report)
-        self.assertIn("| Tests | build | 10 | 3 | 30.0% |", report)
-        self.assertIn("**Total job runs:** 10", report)
-        self.assertIn("**Total failures:** 3", report)
-        self.assertIn("**Overall failure rate:** 30.0%", report)
-
-    @patch("ci_health_report.post_comment")
-    @patch("ci_health_report.get_jobs")
-    @patch("ci_health_report.get_runs")
-    def test_skipped_and_cancelled_not_counted(self, mock_runs, mock_jobs, mock_comment):
-        """skipped and cancelled conclusions are excluded from run and failure counts."""
-        mock_runs.return_value = [{"id": 1, "name": "Tests"}]
-        mock_jobs.return_value = [
-            {"name": "build", "conclusion": "success"},
-            {"name": "build", "conclusion": "failure"},
-            {"name": "build", "conclusion": "skipped"},
-            {"name": "build", "conclusion": "cancelled"},
-        ]
-        with patch.dict("os.environ", {"GH_TOKEN": "tok", "GH_REPO": "o/r", "REPORT_ISSUE": "1", "LOOKBACK_DAYS": "30", "TOP_JOBS": "5"}):
-            main()
-        report = mock_comment.call_args[0][3]
-        self.assertIn("| Tests | build | 2 | 1 |", report)
-
 
 if __name__ == "__main__":
     main()

.github/scripts/ci_health_report/simulate_report.py

@@ -0,0 +1,64 @@
+#!/usr/bin/env python3
+"""
+Generates a sample CI health report with synthetic data and writes it to a file.
+Usage: python3 simulate_report.py [output.md]
+"""
+
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+
+from ci_health_report import build_report
+
+# Each entry: (workflow, job, buckets)
+# Buckets run oldest → newest; each is {runs, failures}.
+SCENARIOS = [
+    # Clearly getting worse — failure rate climbing week over week
+    ("Tests", "lint",        [{"runs": 20, "failures": 1},
+                              {"runs": 20, "failures": 3},
+                              {"runs": 20, "failures": 8},
+                              {"runs": 20, "failures": 14}]),
+    # Clearly improving — failure rate falling
+    ("Tests", "unit",        [{"runs": 20, "failures": 12},
+                              {"runs": 20, "failures": 8},
+                              {"runs": 20, "failures": 3},
+                              {"runs": 20, "failures": 1}]),
+    # Flat / stable low failure rate
+    ("Tests", "build",       [{"runs": 20, "failures": 2},
+                              {"runs": 20, "failures": 2},
+                              {"runs": 20, "failures": 2},
+                              {"runs": 20, "failures": 2}]),
+    # Flat / stable high failure rate
+    ("Integration", "smoke", [{"runs": 20, "failures": 14},
+                              {"runs": 20, "failures": 15},
+                              {"runs": 20, "failures": 13},
+                              {"runs": 20, "failures": 14}]),
+    # Spike in the middle, now recovering
+    ("Integration", "full",  [{"runs": 20, "failures": 2},
+                              {"runs": 20, "failures": 18},
+                              {"runs": 20, "failures": 18},
+                              {"runs": 20, "failures": 3}]),
+    # Sparse — only 2 runs total, should show —
+    ("Nightly", "deploy",    [{"runs": 1,  "failures": 1},
+                              {"runs": 0,  "failures": 0},
+                              {"runs": 0,  "failures": 0},
+                              {"runs": 1,  "failures": 0}]),
+]
+
+stats = defaultdict(lambda: {"runs": 0, "failures": 0, "buckets": []})
+for workflow, job, buckets in SCENARIOS:
+    key = (workflow, job)
+    stats[key]["runs"]     = sum(b["runs"]     for b in buckets)
+    stats[key]["failures"] = sum(b["failures"] for b in buckets)
+    stats[key]["buckets"]  = buckets
+
+now = datetime(2026, 4, 7, 9, 0, 0, tzinfo=timezone.utc)
+report = build_report(stats, lookback_days=30, top_n=5, now=now)
+
+output = sys.argv[1] if len(sys.argv) > 1 else "sample_report.md"
+with open(output, "w") as f:
+    f.write(report + "\n")
+
+print(f"Written to {output}")
+print()
+print(report)

.github/scripts/ci_health_report/test_ci_health_report.py

@@ -0,0 +1,114 @@
+import unittest
+from unittest.mock import MagicMock, patch
+from collections import defaultdict
+from datetime import datetime, timezone
+import urllib.request
+import urllib.error
+
+import ci_health_report
+from ci_health_report import (
+    bucket_count,
+    trend_indicator,
+    build_report,
+    main,
+    _urlopen,
+)
+
+
+class _Tests(unittest.TestCase):
+
+    @patch("time.sleep")
+    @patch("urllib.request.urlopen")
+    def test_rate_limit_retries_with_wait(self, mock_urlopen, mock_sleep):
+        """_urlopen sleeps Retry-After + 5s on 429 then retries successfully."""
+        import http.client
+        msg = http.client.HTTPMessage()
+        msg["Retry-After"] = "10"
+        resp = MagicMock()
+        resp.read.return_value = b'{"ok": true}'
+        resp.__enter__ = lambda s: s
+        resp.__exit__ = MagicMock(return_value=False)
+        mock_urlopen.side_effect = [
+            urllib.error.HTTPError("https://api.github.com/test", 429, "Too Many Requests", msg, None),
+            resp,
+        ]
+        result = _urlopen(urllib.request.Request("https://api.github.com/test"))
+        mock_sleep.assert_called_once_with(15)  # Retry-After(10) + 5
+        self.assertEqual(result, b'{"ok": true}')
+
+    def test_bucket_count(self):
+        """bucket_count returns daily, weekly, or monthly bucket counts."""
+        self.assertEqual(bucket_count(7),  7)   # daily
+        self.assertEqual(bucket_count(14), 14)  # daily (boundary)
+        self.assertEqual(bucket_count(30), 4)   # weekly
+        self.assertEqual(bucket_count(90), 12)  # weekly (boundary)
+        self.assertEqual(bucket_count(91), 3)   # monthly
+
+    def test_trend_indicator_increasing(self):
+        """trend_indicator returns ↑ when recent half has higher failure rate."""
+        buckets = [
+            {"runs": 10, "failures": 1},
+            {"runs": 10, "failures": 1},
+            {"runs": 10, "failures": 5},
+            {"runs": 10, "failures": 5},
+        ]
+        result = trend_indicator(buckets)
+        self.assertTrue(result.startswith("↑"))
+
+    def test_trend_indicator_decreasing(self):
+        """trend_indicator returns ↓ when recent half has lower failure rate."""
+        buckets = [
+            {"runs": 10, "failures": 5},
+            {"runs": 10, "failures": 5},
+            {"runs": 10, "failures": 1},
+            {"runs": 10, "failures": 1},
+        ]
+        result = trend_indicator(buckets)
+        self.assertTrue(result.startswith("↓"))
+
+    def test_build_report_trend_shown_when_sufficient_runs(self):
+        """build_report shows trend arrow when runs >= num_buckets * 2."""
+        buckets = [{"runs": 5, "failures": 1}, {"runs": 5, "failures": 4}]  # 10 runs, threshold=4
+        stats = defaultdict(lambda: {"runs": 0, "failures": 0, "buckets": []})
+        stats[("Tests", "build")]["runs"] = 10
+        stats[("Tests", "build")]["failures"] = 5
+        stats[("Tests", "build")]["buckets"] = buckets
+        now = datetime(2026, 1, 1, 9, 0, 0, tzinfo=timezone.utc)
+        report = build_report(stats, 30, 5, now)
+        self.assertIn("| Workflow | Job | Runs | Failures | Rate | Trend |", report)
+        self.assertIn("| Tests | build | 10 | 5 | 50.0% | ↑", report)
+        self.assertIn("**Total job runs:** 10", report)
+        self.assertIn("**Overall failure rate:** 50.0%", report)
+
+    def test_build_report_trend_suppressed_when_sparse(self):
+        """build_report shows — for trend when runs < num_buckets * 2."""
+        buckets = [{"runs": 1, "failures": 1}, {"runs": 0, "failures": 0},
+                   {"runs": 0, "failures": 0}, {"runs": 1, "failures": 0}]
+        stats = defaultdict(lambda: {"runs": 0, "failures": 0, "buckets": []})
+        stats[("Nightly", "deploy")]["runs"] = 2
+        stats[("Nightly", "deploy")]["failures"] = 1
+        stats[("Nightly", "deploy")]["buckets"] = buckets
+        now = datetime(2026, 1, 1, 9, 0, 0, tzinfo=timezone.utc)
+        report = build_report(stats, 30, 5, now)
+        self.assertIn("| Nightly | deploy | 2 | 1 | 50.0% | — |", report)
+
+    @patch("ci_health_report.post_comment")
+    @patch("ci_health_report.get_jobs")
+    @patch("ci_health_report.get_runs")
+    def test_skipped_and_cancelled_not_counted(self, mock_runs, mock_jobs, mock_comment):
+        """skipped and cancelled conclusions are excluded from run and failure counts."""
+        mock_runs.return_value = [{"id": 1, "name": "Tests", "created_at": "2026-01-15T00:00:00Z"}]
+        mock_jobs.return_value = [
+            {"name": "build", "conclusion": "success"},
+            {"name": "build", "conclusion": "failure"},
+            {"name": "build", "conclusion": "skipped"},
+            {"name": "build", "conclusion": "cancelled"},
+        ]
+        with patch.dict("os.environ", {"GH_TOKEN": "tok", "GH_REPO": "o/r", "REPORT_ISSUE": "1", "LOOKBACK_DAYS": "30", "TOP_JOBS": "5"}):
+            main()
+        report = mock_comment.call_args[0][3]
+        self.assertIn("| Tests | build | 2 | 1 |", report)
+
+
+if __name__ == "__main__":
+    unittest.main()