Merge pull request #628 from lbedner/insights-service

Insights Service

Author: lbedner

.github/workflows/security.yml

@@ -52,9 +52,11 @@ jobs:
 
     - name: Run security audit
       run: |
-        # Ignoring pip 25.2 vulnerabilities (fixed in pip 26.0, not yet released)
-        # GHSA-4xh5-x5gv-qwph: symlink traversal vulnerability
-        # GHSA-6vgw-5pg2-w6jp: additional pip vulnerability
+        # Ignoring pip 25.2 vulnerabilities (uv manages pip, not user-facing)
         # Risk: Low - only affects installation of malicious packages from untrusted sources
         # Mitigation: All packages installed from trusted PyPI with uv.lock verification
-        uv run pip-audit --ignore-vuln GHSA-4xh5-x5gv-qwph --ignore-vuln GHSA-6vgw-5pg2-w6jp
+        uv run pip-audit \
+          --ignore-vuln GHSA-4xh5-x5gv-qwph \
+          --ignore-vuln GHSA-6vgw-5pg2-w6jp \
+          --ignore-vuln ECHO-ffe1-1d3c-d9bc \
+          --ignore-vuln ECHO-7db2-03aa-5591

README.md

@@ -132,6 +132,7 @@ Most starters lock you in at `init`. Aegis Stack doesn't. See **[Evolving Your S
 - **Auth** → JWT authentication
 - **AI** → PydanticAI / LangChain
 - **Comms** → Resend + Twilio
+- **Insights** → Adoption metrics (GitHub, PyPI, Plausible) *(experimental)*
 
 [Components Docs →](https://lbedner.github.io/aegis-stack/components/) | [Services Docs →](https://lbedner.github.io/aegis-stack/services/)
 

aegis/cli/callbacks.py

@@ -21,6 +21,10 @@
     restore_engine_info,
 )
 from ..core.dependency_resolver import DependencyResolver
+from ..core.insights_service_parser import (
+    is_insights_service_with_options,
+    parse_insights_service_config,
+)
 from ..core.service_resolver import ServiceResolver
 from ..core.services import SERVICES
 from ..i18n import t
@@ -212,6 +216,18 @@ def validate_and_resolve_services(
                 typer.secho(f"Invalid auth service syntax: {e}", fg="red", err=True)
                 raise typer.Exit(1)
 
+    # Parse Insights service bracket syntax
+    for service in selected_services:
+        if is_insights_service_with_options(service):
+            try:
+                insights_config = parse_insights_service_config(service)
+                typer.echo(
+                    f"Insights service: sources={','.join(insights_config.sources)}"
+                )
+            except ValueError as e:
+                typer.secho(f"Invalid insights service syntax: {e}", fg="red", err=True)
+                raise typer.Exit(1)
+
     # Resolve services to components
     resolved_components, service_added = ServiceResolver.resolve_service_dependencies(
         selected_services

aegis/commands/add_service.py

@@ -422,6 +422,24 @@ def add_service_command(
                 if isinstance(framework, str):
                     service_data[AnswerKeys.AI_FRAMEWORK] = framework
 
+            # For insights service, pass source flags
+            if base_service == AnswerKeys.SERVICE_INSIGHTS:
+                from ..core.insights_service_parser import (
+                    DEFAULT_SOURCES,
+                    is_insights_service_with_options,
+                    parse_insights_service_config,
+                )
+
+                if is_insights_service_with_options(service):
+                    insights_config = parse_insights_service_config(service)
+                    sources = insights_config.sources
+                else:
+                    sources = DEFAULT_SOURCES
+                service_data[AnswerKeys.INSIGHTS_GITHUB] = "github" in sources
+                service_data[AnswerKeys.INSIGHTS_PYPI] = "pypi" in sources
+                service_data[AnswerKeys.INSIGHTS_PLAUSIBLE] = "plausible" in sources
+                service_data[AnswerKeys.INSIGHTS_REDDIT] = "reddit" in sources
+
             # Add the service (services are added like components)
             # Use base_service for file lookup, not the full variant name
             result = updater.add_component(base_service, service_data)

aegis/constants.py

@@ -125,11 +125,19 @@ class AnswerKeys:
     AUTH = "include_auth"
     AI = "include_ai"
     COMMS = "include_comms"
+    INSIGHTS = "include_insights"
 
     # Service names (used for selection/lookup)
     SERVICE_AUTH = "auth"
     SERVICE_AI = "ai"
     SERVICE_COMMS = "comms"
+    SERVICE_INSIGHTS = "insights"
+
+    # Insights source flags
+    INSIGHTS_GITHUB = "insights_github"
+    INSIGHTS_PYPI = "insights_pypi"
+    INSIGHTS_PLAUSIBLE = "insights_plausible"
+    INSIGHTS_REDDIT = "insights_reddit"
 
     # Configuration values
     SCHEDULER_BACKEND = "scheduler_backend"

aegis/core/copier_manager.py

@@ -138,6 +138,21 @@ def generate_with_copier(
         AnswerKeys.AI_RAG: template_context.get(AnswerKeys.AI_RAG, "no") == "yes",
         AnswerKeys.AI_VOICE: template_context.get(AnswerKeys.AI_VOICE, "no") == "yes",
         AnswerKeys.OLLAMA_MODE: template_context.get(AnswerKeys.OLLAMA_MODE, "none"),
+        AnswerKeys.INSIGHTS: template_context.get(AnswerKeys.INSIGHTS, "no") == "yes",
+        AnswerKeys.INSIGHTS_GITHUB: template_context.get(
+            AnswerKeys.INSIGHTS_GITHUB, "no"
+        )
+        == "yes",
+        AnswerKeys.INSIGHTS_PYPI: template_context.get(AnswerKeys.INSIGHTS_PYPI, "no")
+        == "yes",
+        AnswerKeys.INSIGHTS_PLAUSIBLE: template_context.get(
+            AnswerKeys.INSIGHTS_PLAUSIBLE, "no"
+        )
+        == "yes",
+        AnswerKeys.INSIGHTS_REDDIT: template_context.get(
+            AnswerKeys.INSIGHTS_REDDIT, "no"
+        )
+        == "yes",
     }
 
     # Detect dev vs production mode for template sourcing
@@ -228,6 +243,7 @@ def generate_with_copier(
     # This ensures consistent behavior with Cookiecutter
     include_auth = copier_data.get(AnswerKeys.AUTH, False)
     include_ai = copier_data.get(AnswerKeys.AI, False)
+    include_insights = copier_data.get(AnswerKeys.INSIGHTS, False)
     ai_backend = copier_data.get(AnswerKeys.AI_BACKEND, StorageBackends.MEMORY)
     database_engine = copier_data.get(
         AnswerKeys.DATABASE_ENGINE, StorageBackends.SQLITE
@@ -240,8 +256,11 @@ def generate_with_copier(
     # Type narrowing: ai_backend should always be a string, but narrow from Any
     ai_backend_str: str = str(ai_backend) if ai_backend else StorageBackends.MEMORY
 
+    is_insights_included: bool = include_insights is True
     ai_needs_migrations = is_ai_included and ai_backend_str != StorageBackends.MEMORY
-    needs_migration_files = is_auth_included or ai_needs_migrations
+    needs_migration_files = (
+        is_auth_included or ai_needs_migrations or is_insights_included
+    )
     # Only run migrations automatically for SQLite (file-based, no server needed)
     # PostgreSQL requires a running server, so skip auto-migration
     is_sqlite = database_engine == StorageBackends.SQLITE

aegis/core/insights_service_parser.py

@@ -0,0 +1,105 @@
+"""
+Insights service bracket syntax parser.
+
+Parses insights[sources...] syntax where values are data source names:
+- github: GitHub Traffic API + Stargazers API
+- pypi: PyPI/pepy.tech download stats
+- plausible: Plausible docs analytics
+- reddit: Reddit post tracking
+
+Order doesn't matter. Defaults: github, pypi
+"""
+
+from dataclasses import dataclass, field
+
+# Valid source names
+SOURCES = {"github", "pypi", "plausible", "reddit"}
+
+# Default sources when no brackets specified
+DEFAULT_SOURCES = ["github", "pypi"]
+
+
+@dataclass
+class InsightsServiceConfig:
+    """Parsed insights service configuration."""
+
+    sources: list[str] = field(default_factory=lambda: DEFAULT_SOURCES.copy())
+
+
+def parse_insights_service_config(service_string: str) -> InsightsServiceConfig:
+    """
+    Parse insights[...] service string into config.
+
+    Args:
+        service_string: Service specification like "insights", "insights[github]",
+                        or "insights[github,pypi,plausible,reddit]"
+
+    Returns:
+        InsightsServiceConfig with selected sources
+
+    Raises:
+        ValueError: If service string is invalid or has unknown values
+    """
+    service_string = service_string.strip()
+
+    if not service_string.startswith("insights"):
+        raise ValueError(
+            f"Expected 'insights' service, got '{service_string}'. "
+            "This parser only handles insights[...] syntax."
+        )
+
+    # Plain "insights" with no brackets
+    if service_string == "insights":
+        return InsightsServiceConfig()
+
+    if "[" not in service_string:
+        raise ValueError(
+            f"Invalid service string '{service_string}'. "
+            "Expected 'insights' or 'insights[sources]' format."
+        )
+
+    if not service_string.endswith("]"):
+        raise ValueError(
+            f"Malformed brackets in '{service_string}'. Expected closing ']'."
+        )
+
+    bracket_start = service_string.index("[")
+    bracket_content = service_string[bracket_start + 1 : -1].strip()
+
+    # Empty brackets = defaults
+    if not bracket_content:
+        return InsightsServiceConfig()
+
+    # Split by comma and validate
+    values = [v.strip().lower() for v in bracket_content.split(",") if v.strip()]
+
+    # Check for duplicates
+    seen: set[str] = set()
+    for value in values:
+        if value in seen:
+            raise ValueError(f"Duplicate source '{value}' in insights[...] syntax.")
+        seen.add(value)
+
+        if value not in SOURCES:
+            raise ValueError(
+                f"Unknown source '{value}' in insights[...] syntax. "
+                f"Valid sources: {', '.join(sorted(SOURCES))}."
+            )
+
+    return InsightsServiceConfig(sources=values)
+
+
+def is_insights_service_with_options(service_string: str) -> bool:
+    """
+    Check if a service string is an insights service with bracket options.
+
+    Returns True ONLY when explicit bracket syntax is used (insights[...]).
+    Plain "insights" without brackets returns False.
+
+    Args:
+        service_string: Service specification string
+
+    Returns:
+        True if this is an insights[...] format string with explicit options
+    """
+    return service_string.strip().startswith("insights[")

aegis/core/post_gen_tasks.py

@@ -185,6 +185,17 @@ def get_component_file_mapping() -> dict[str, list[str]]:
             "tests/api/test_voice_endpoints.py",
             "app/components/frontend/dashboard/modals/voice_settings_tab.py",
         ],
+        AnswerKeys.SERVICE_INSIGHTS: [
+            "app/components/backend/api/insights",
+            "app/services/insights",
+            "app/cli/insights.py",
+            "tests/services/test_insights_service.py",
+            "tests/services/test_insights_collectors.py",
+            "tests/api/test_insights_endpoints.py",
+            # Frontend dashboard files
+            "app/components/frontend/dashboard/cards/insights_card.py",
+            "app/components/frontend/dashboard/modals/insights_modal.py",
+        ],
     }
 
 
@@ -559,6 +570,21 @@ def _rename_backend_files(suffix: str) -> set[str]:
             project_path, "app/components/frontend/dashboard/modals/comms_modal.py"
         )
 
+    # Remove insights service if not selected
+    if not is_enabled(AnswerKeys.INSIGHTS):
+        remove_dir(project_path, "app/components/backend/api/insights")
+        remove_dir(project_path, "app/services/insights")
+        remove_file(project_path, "app/cli/insights.py")
+        remove_file(project_path, "tests/services/test_insights_service.py")
+        remove_file(project_path, "tests/services/test_insights_collectors.py")
+        remove_file(project_path, "tests/api/test_insights_endpoints.py")
+        remove_file(
+            project_path, "app/components/frontend/dashboard/cards/insights_card.py"
+        )
+        remove_file(
+            project_path, "app/components/frontend/dashboard/modals/insights_modal.py"
+        )
+
     # Remove auth service dashboard files if not selected
     if not is_enabled(AnswerKeys.AUTH):
         remove_file(
@@ -569,23 +595,25 @@ def _rename_backend_files(suffix: str) -> set[str]:
         )
 
     # Remove services_card.py only if NO services are enabled
-    # ServicesCard shows all services (auth, AI, comms), so keep if ANY service is enabled
+    # ServicesCard shows all services, so keep if ANY service is enabled
     if (
         not is_enabled(AnswerKeys.AUTH)
         and not is_enabled(AnswerKeys.AI)
         and not is_enabled(AnswerKeys.COMMS)
+        and not is_enabled(AnswerKeys.INSIGHTS)
     ):
         remove_file(
             project_path, "app/components/frontend/dashboard/cards/services_card.py"
         )
 
     # Remove Alembic directory only if NO service needs migrations
-    # Alembic is needed when: auth is enabled OR (AI is enabled AND backend is NOT memory)
+    # Alembic is needed when: auth, insights, or (AI with non-memory backend)
     include_auth = is_enabled(AnswerKeys.AUTH)
     include_ai = is_enabled(AnswerKeys.AI)
+    include_insights = is_enabled(AnswerKeys.INSIGHTS)
     ai_backend = context.get(AnswerKeys.AI_BACKEND, StorageBackends.MEMORY)
     ai_needs_migrations = include_ai and ai_backend != StorageBackends.MEMORY
-    needs_migrations = include_auth or ai_needs_migrations
+    needs_migrations = include_auth or ai_needs_migrations or include_insights
 
     if not needs_migrations:
         remove_dir(project_path, "alembic")
@@ -632,6 +660,7 @@ def _render_jinja_template(src: Path, dst: Path, project_path: Path) -> None:
         "include_auth": True,
         "include_ai": True,
         "include_comms": True,
+        "include_insights": True,
         # Component flags - check what exists in project
         "include_scheduler": (project_path / "app/components/scheduler").exists(),
         "include_worker": (project_path / "app/components/worker").exists(),

aegis/core/service_resolver.py

@@ -11,6 +11,10 @@
 from .auth_service_parser import is_auth_service_with_options, parse_auth_service_config
 from .component_utils import extract_base_component_name, extract_base_service_name
 from .dependency_resolver import DependencyResolver
+from .insights_service_parser import (
+    is_insights_service_with_options,
+    parse_insights_service_config,
+)
 from .services import SERVICES, get_service_dependencies
 
 
@@ -126,6 +130,16 @@ def validate_services(services: list[str]) -> list[str]:
                 except ValueError as e:
                     errors.append(f"Invalid AI service syntax: {e}")
 
+            # Validate insights service bracket syntax if provided
+            if (
+                base_service == AnswerKeys.SERVICE_INSIGHTS
+                and is_insights_service_with_options(service)
+            ):
+                try:
+                    parse_insights_service_config(service)
+                except ValueError as e:
+                    errors.append(f"Invalid insights service syntax: {e}")
+
             spec = SERVICES[base_service]
 
             # Check service conflicts