mcp-prerelease-compatibility.yml

name: MCP Prerelease Compatibility Testing

on:
  schedule:
    # Run daily at 2 AM UTC to catch new prereleases
    - cron: '0 2 * * *'
  workflow_dispatch:
    inputs:
      test_specific_version:
        description: 'Specific version to test (optional)'
        required: false
        type: string

jobs:
  discover-mcp-prereleases:
    runs-on: ubuntu-latest
    outputs:
      mcp-prereleases: ${{ steps.get-mcp-prereleases.outputs.versions }}
      has-prereleases: ${{ steps.get-mcp-prereleases.outputs.has-prereleases }}

    steps:
    - name: Get available MCP prerelease versions
      id: get-mcp-prereleases
      run: |
        # Get all available versions from PyPI including prereleases
        all_versions=$(pip index versions mcp --pre 2>/dev/null | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+[ab][0-9]\+\|[0-9]\+\.[0-9]\+\.[0-9]\+rc[0-9]\+' | sort -V || echo "")

        if [ -z "$all_versions" ]; then
          echo "No MCP prereleases found"
          echo "versions=[]" >> $GITHUB_OUTPUT
          echo "has-prereleases=false" >> $GITHUB_OUTPUT
        else
          # Get only the latest prerelease
          latest=$(echo "$all_versions" | tail -n 1)
          json_array="[\"$latest\"]"
          echo "Found MCP prerelease: $json_array"
          echo "versions=$json_array" >> $GITHUB_OUTPUT
          echo "has-prereleases=true" >> $GITHUB_OUTPUT
        fi

  discover-fastmcp-prereleases:
    runs-on: ubuntu-latest
    outputs:
      fastmcp-prereleases: ${{ steps.get-fastmcp-prereleases.outputs.versions }}
      has-prereleases: ${{ steps.get-fastmcp-prereleases.outputs.has-prereleases }}

    steps:
    - name: Get available FastMCP prerelease versions
      id: get-fastmcp-prereleases
      run: |
        # Get all available versions from PyPI including prereleases
        all_versions=$(pip index versions fastmcp --pre 2>/dev/null | grep -o '[0-9]\+\.[0-9]\+\.[0-9]\+[ab][0-9]\+\|[0-9]\+\.[0-9]\+\.[0-9]\+rc[0-9]\+' | sort -V || echo "")

        if [ -z "$all_versions" ]; then
          echo "No FastMCP prereleases found"
          echo "versions=[]" >> $GITHUB_OUTPUT
          echo "has-prereleases=false" >> $GITHUB_OUTPUT
        else
          # Get only the latest prerelease
          latest=$(echo "$all_versions" | tail -n 1)
          json_array="[\"$latest\"]"
          echo "Found FastMCP prerelease: $json_array"
          echo "versions=$json_array" >> $GITHUB_OUTPUT
          echo "has-prereleases=true" >> $GITHUB_OUTPUT
        fi

  test-fastmcp-prerelease:
    runs-on: ubuntu-latest
    needs: discover-fastmcp-prereleases
    if: needs.discover-fastmcp-prereleases.outputs.has-prereleases == 'true'
    strategy:
      matrix:
        fastmcp-version: ${{ fromJson(needs.discover-fastmcp-prereleases.outputs.fastmcp-prereleases) }}
      fail-fast: false

    steps:
    - uses: actions/checkout@v4

    - name: Install uv
      uses: astral-sh/setup-uv@v4
      with:
        version: "latest"

    - name: Set up Python 3.12
      uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Update pyproject.toml with prerelease FastMCP ${{ matrix.fastmcp-version }}
      run: |
        # Update the community extras section with the specific FastMCP prerelease version
        sed -i -E 's/"fastmcp[>=]+[^"]+"/\"fastmcp==${{ matrix.fastmcp-version }}\"/' pyproject.toml

        # Keep MCP flexible so FastMCP can choose its version
        sed -i -E 's/"mcp[>=]+[^"]+"/\"mcp>=1.2.0\"/' pyproject.toml

        # Show the updated dependencies for verification
        echo "Updated dependencies:"
        grep -A 5 "^dependencies = \[" pyproject.toml
        echo ""
        echo "Updated community extras:"
        grep -A 2 "^\[project.optional-dependencies\]" pyproject.toml | grep -A 2 "community"

    - name: Install dependencies with prerelease FastMCP ${{ matrix.fastmcp-version }}
      run: |
        # Install with both dev and community extras, allow prereleases
        # FastMCP will determine which MCP version it needs
        uv sync --extra dev --extra community --prerelease allow

    - name: Show installed versions
      run: |
        echo "Installed package versions:"
        uv pip show mcp | grep "Version:" || echo "MCP not found"
        uv pip show fastmcp | grep "Version:" || echo "FastMCP not found"
        echo ""
        echo "Full MCP package info:"
        uv pip show mcp || echo "MCP package details not available"

    - name: Run community FastMCP tests with prerelease version ${{ matrix.fastmcp-version }}
      run: |
        echo "Running community FastMCP tests with prerelease version ${{ matrix.fastmcp-version }}"
        set -o pipefail
        uv run pytest -v 2>&1 | tee pytest.log

    - name: Collect failure details
      if: failure()
      env:
        PACKAGE_UNDER_TEST: fastmcp
        PACKAGE_VERSION: ${{ matrix.fastmcp-version }}
      run: |
        python .github/scripts/collect_failure.py pytest.log

    - name: Upload failure artifact
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: prerelease-failure-fastmcp-${{ matrix.fastmcp-version }}
        path: failure.json
        retention-days: 7

  test-mcp-prerelease:
    runs-on: ubuntu-latest
    needs: discover-mcp-prereleases
    if: needs.discover-mcp-prereleases.outputs.has-prereleases == 'true'
    strategy:
      matrix:
        mcp-version: ${{ fromJson(needs.discover-mcp-prereleases.outputs.mcp-prereleases) }}
      fail-fast: false

    steps:
    - uses: actions/checkout@v4

    - name: Install uv
      uses: astral-sh/setup-uv@v4
      with:
        version: "latest"

    - name: Set up Python 3.12
      uses: actions/setup-python@v5
      with:
        python-version: '3.12'

    - name: Install dependencies with prerelease MCP ${{ matrix.mcp-version }} (no FastMCP)
      run: |
        # Create virtual environment
        uv venv

        # Install base dependencies first
        uv pip install pydantic>=2.0.0 requests>=2.31.0 mcpcat-api==0.1.4

        # Install the specific MCP prerelease version
        uv pip install "mcp==${{ matrix.mcp-version }}" --prerelease=allow

        # Install dev dependencies for testing
        uv pip install pytest>=7.0.0 pytest-asyncio>=0.21.0 mypy>=1.0.0 ruff>=0.1.0 freezegun>=1.2.0

        # Explicitly uninstall fastmcp if it somehow got installed
        uv pip uninstall fastmcp -y 2>/dev/null || true

        # Install the package itself in editable mode
        uv pip install -e .

    - name: Verify FastMCP is not installed
      run: |
        if uv pip show fastmcp 2>/dev/null; then
          echo "ERROR: fastmcp should not be installed for MCP-only tests"
          exit 1
        else
          echo "✓ Confirmed: fastmcp is not installed"
        fi

    - name: Show installed MCP version
      run: |
        echo "Installed MCP version:"
        uv pip show mcp | grep "Version:" || echo "MCP not found"
        echo ""
        echo "Full MCP package info:"
        uv pip show mcp || echo "MCP package details not available"

    - name: Run tests with prerelease MCP ${{ matrix.mcp-version }}
      run: |
        echo "Running tests with prerelease MCP version ${{ matrix.mcp-version }} (no FastMCP)"
        # Run all tests except community tests
        set -o pipefail
        uv run pytest -v 2>&1 | tee pytest.log

    - name: Collect failure details
      if: failure()
      env:
        PACKAGE_UNDER_TEST: mcp
        PACKAGE_VERSION: ${{ matrix.mcp-version }}
      run: |
        python .github/scripts/collect_failure.py pytest.log

    - name: Upload failure artifact
      if: failure()
      uses: actions/upload-artifact@v4
      with:
        name: prerelease-failure-mcp-${{ matrix.mcp-version }}
        path: failure.json
        retention-days: 7

  send-failure-notification:
    runs-on: ubuntu-latest
    needs: [discover-mcp-prereleases, discover-fastmcp-prereleases, test-fastmcp-prerelease, test-mcp-prerelease]
    if: failure()

    steps:
    - name: Download failure details
      uses: actions/download-artifact@v4
      with:
        pattern: prerelease-failure-*
        path: failure_artifacts
        merge-multiple: true
        if-no-files-found: ignore

    - name: Send failure notification email
      run: |
        # Install resend
        pip install resend

        # Send email notification
        python -c "
        import resend
        import os
        import sys
        import json
        from pathlib import Path

        # Get sensitive data from environment
        api_key = os.environ.get('RESEND_API_KEY')
        alert_email = os.environ.get('ALERT_EMAIL')

        if not api_key:
            print('Error: RESEND_API_KEY secret not configured')
            sys.exit(1)

        if not alert_email:
            print('Error: ALERT_EMAIL secret not configured')
            sys.exit(1)

        # Set API key without printing it
        resend.api_key = api_key

        # Determine which tests failed
        workflow_url = 'https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}'

        # Get the prerelease versions that were being tested
        mcp_prereleases = '${{ needs.discover-mcp-prereleases.outputs.mcp-prereleases }}'
        fastmcp_prereleases = '${{ needs.discover-fastmcp-prereleases.outputs.fastmcp-prereleases }}'
        has_mcp_prereleases = '${{ needs.discover-mcp-prereleases.outputs.has-prereleases }}'
        has_fastmcp_prereleases = '${{ needs.discover-fastmcp-prereleases.outputs.has-prereleases }}'

        def parse_versions(raw: str, label: str, enabled: bool) -> list[str]:
            raw = (raw or '').strip()
            if not raw or not enabled:
                return []

            try:
                parsed = json.loads(raw)
            except json.JSONDecodeError as exc:
                print(f'Warning: Could not parse {label} JSON: {exc}')
                cleaned = raw.strip('[]')
                if not cleaned:
                    return []
                items = []
                for piece in cleaned.split(','):
                    candidate = piece.strip()
                    candidate = candidate.strip(chr(39))
                    candidate = candidate.strip(chr(34))
                    if candidate:
                        items.append(candidate)
                return items

            if isinstance(parsed, list):
                return [str(item).strip() for item in parsed if str(item).strip()]

            print(
                f'Warning: Unexpected {label} payload type: '
                f'{type(parsed).__name__}'
            )
            return []

        mcp_list = parse_versions(
            mcp_prereleases,
            'MCP prereleases',
            has_mcp_prereleases == 'true'
        )
        fastmcp_list = parse_versions(
            fastmcp_prereleases,
            'FastMCP prereleases',
            has_fastmcp_prereleases == 'true'
        )
        print(
            f'Using {len(mcp_list)} MCP prereleases and '
            f'{len(fastmcp_list)} FastMCP prereleases'
        )

        # Format version lists
        mcp_versions_html = '<br>'.join([f'  • MCP {v}' for v in mcp_list]) if mcp_list else 'No MCP prereleases tested'
        fastmcp_versions_html = '<br>'.join([f'  • FastMCP {v}' for v in fastmcp_list]) if fastmcp_list else 'No FastMCP prereleases tested'

        failure_dir = Path('failure_artifacts')
        failure_details = []
        if failure_dir.exists():
            for failure_path in failure_dir.rglob('failure.json'):
                try:
                    failure_payload = json.loads(failure_path.read_text())
                except Exception as exc:
                    print(f'Warning: Could not parse failure file {failure_path}: {exc}')
                    continue
                failure_payload['_source'] = str(failure_path)
                failure_details.append(failure_payload)
        print(f'Collected {len(failure_details)} failure detail files')

        def failing_versions(package_name: str) -> list[str]:
            versions = []
            for entry in failure_details:
                package_value = str(entry.get('package', '')).lower()
                if package_value != package_name.lower():
                    continue
                version_value = str(entry.get('version', '')).strip()
                if version_value and version_value not in versions:
                    versions.append(version_value)
            return versions

        failing_mcp = failing_versions('mcp')
        failing_fastmcp = failing_versions('fastmcp')

        failing_sections = []
        if failing_mcp:
            failing_sections.append(
                '<p><strong>MCP Prerelease Versions Failing:</strong><br>'
                + '<br>'.join([f'  • MCP {version}' for version in failing_mcp])
                + '</p>'
            )
        if failing_fastmcp:
            failing_sections.append(
                '<p><strong>FastMCP Prerelease Versions Failing:</strong><br>'
                + '<br>'.join([f'  • FastMCP {version}' for version in failing_fastmcp])
                + '</p>'
            )
        if not failing_sections:
            if failure_details:
                failing_sections.append(
                    '<p>Failures detected but version details were unavailable.</p>'
                )
            else:
                failing_sections.append(
                    '<p>No failure artifacts were collected. Review the workflow logs for more information.</p>'
                )
        failing_versions_html = '\n'.join(failing_sections)

        subject_parts: list[str] = []
        if failing_mcp:
            subject_parts.append('MCP ' + ', '.join(failing_mcp))
        if failing_fastmcp:
            subject_parts.append('FastMCP ' + ', '.join(failing_fastmcp))
        subject_suffix = ' | '.join(subject_parts) if subject_parts else 'See workflow logs'
        subject_line = f'⚠️ MCPCat Prerelease Tests Failed - {subject_suffix}'
        print(f'Email subject suffix: {subject_suffix}')

        email_html = f'''
        <h2>⚠️ MCPCat Prerelease Compatibility Tests Failed</h2>

        <p>The prerelease compatibility tests have failed. This may indicate upcoming breaking changes in MCP or FastMCP.</p>

        <h3>Details:</h3>
        <ul>
          <li>Workflow: ${{ github.workflow }}</li>
          <li>Repository: ${{ github.repository }}</li>
          <li>Branch: ${{ github.ref }}</li>
          <li>Run ID: ${{ github.run_id }}</li>
        </ul>

        <p><a href=\"{workflow_url}\">View full workflow run details</a></p>

        <h3>Failing Versions:</h3>
        {failing_versions_html}

        <h3>Prerelease Versions Tested:</h3>
        <p><strong>MCP Prerelease Versions:</strong><br>
        {mcp_versions_html}</p>

        <p><strong>FastMCP Prerelease Versions:</strong><br>
        {fastmcp_versions_html}</p>

        <h3>What to check:</h3>
        <ul>
          <li>Review the specific failing versions in the workflow</li>
          <li>Check for deprecation warnings or breaking changes</li>
          <li>Update compatibility if needed before the stable release</li>
          <li>Consider if changes are needed to support upcoming versions</li>
        </ul>
        '''

        # Replace slashes with underscores for Resend tags
        repo_tag = '${{ github.repository }}'.replace('/', '_')

        params = {
            'from': 'no-reply@mcpcat.io',  # Use Resend's default verified domain
            'to': [alert_email],  # Use variable, not inline secret
            'subject': subject_line,
            'html': email_html,
            'tags': [
                {'name': 'alert_type', 'value': 'prerelease_failure'},
                {'name': 'repository', 'value': repo_tag}
            ]
        }

        try:
            # Validate email format
            if not alert_email or '@' not in alert_email:
                print('Error: Invalid email address format')
                sys.exit(1)

            # Send email without logging sensitive params
            email = resend.Emails.send(params)
            # Only print success message, not the email object which might contain sensitive data
            print('Email notification sent successfully')
        except resend.exceptions.ValidationError as e:
            # ValidationError might mean the API key is invalid or email format issue
            print(f'Resend ValidationError - Check that:')
            print('1. RESEND_API_KEY is valid and starts with re_')
            print('2. ALERT_EMAIL is a valid email address')
            print('3. The from address domain is verified in Resend')
            sys.exit(1)
        except Exception as e:
            # Print error type without sensitive details
            print(f'Failed to send email notification: {type(e).__name__}')
            # Exit with error code to indicate failure (but workflow continues)
            sys.exit(1)
        "
      env:
        RESEND_API_KEY: ${{ secrets.RESEND_API_KEY }}
        ALERT_EMAIL: ${{ secrets.ALERT_EMAIL }}

  report-prerelease-compatibility:
    runs-on: ubuntu-latest
    needs: [discover-mcp-prereleases, discover-fastmcp-prereleases, test-fastmcp-prerelease, test-mcp-prerelease, send-failure-notification]
    if: always()

    steps:
    - name: Generate Prerelease Compatibility Report
      run: |
        echo "=========================================="
        echo "   🔬 PRERELEASE COMPATIBILITY REPORT    "
        echo "=========================================="
        echo ""
        echo "⚠️  PRERELEASE VERSION TESTING ⚠️"
        echo "This report shows compatibility with PRERELEASE versions"
        echo "which may be unstable or have breaking changes."
        echo ""
        echo "Generated on: $(date)"
        echo ""

        echo "📦 MCP PRERELEASE STATUS"
        echo "────────────────────────────────────────"
        if [ "${{ needs.discover-mcp-prereleases.outputs.has-prereleases }}" == "true" ]; then
          echo "Prerelease versions tested: ${{ needs.discover-mcp-prereleases.outputs.mcp-prereleases }}"
          echo "Test Status: Check individual job results above"
        else
          echo "No MCP prerelease versions currently available"
        fi
        echo ""

        echo "📦 FASTMCP PRERELEASE STATUS"
        echo "────────────────────────────────────────"
        if [ "${{ needs.discover-fastmcp-prereleases.outputs.has-prereleases }}" == "true" ]; then
          echo "Prerelease versions tested: ${{ needs.discover-fastmcp-prereleases.outputs.fastmcp-prereleases }}"
          echo "Test Status: Check individual job results above"
          echo "Note: FastMCP determines its own MCP dependency version"
        else
          echo "No FastMCP prerelease versions currently available"
        fi
        echo ""

        echo "🔍 TEST CONFIGURATION"
        echo "────────────────────────────────────────"
        echo "• Python version: 3.12"
        echo "• Email alerts: Sent on failure via Resend"
        echo "• MCP dependency: >=1.2.0 (flexible, FastMCP determines actual version)"
        echo ""

        echo "📊 HOW TO INTERPRET RESULTS"
        echo "────────────────────────────────────────"
        echo "✅ PASSED - Prerelease is compatible"
        echo "❌ FAILED - Prerelease has compatibility issues"
        echo ""
        echo "Failed tests will trigger email notifications for early warning"
        echo "about upcoming compatibility changes in MCP/FastMCP prereleases."
        echo ""

        if [ "${{ github.event_name }}" == "workflow_dispatch" ]; then
          echo "📝 MANUAL RUN DETAILS"
          echo "────────────────────────────────────────"
          echo "Triggered manually via workflow_dispatch"
          if [ -n "${{ github.event.inputs.test_specific_version }}" ]; then
            echo "Specific version requested: ${{ github.event.inputs.test_specific_version }}"
          fi
          echo ""
        fi

        echo "=========================================="