_evaluations.yaml

# Reusable workflow for MCP tool calling evaluations.
# Evaluates AI models' ability to correctly identify and call MCP tools.

on:
    workflow_call:
        inputs:
            pr_number:
                description: Pull request number
                required: true
                type: string

jobs:
    evaluations:
        name: MCP tool calling evaluations
        runs-on: ubuntu-latest

        steps:
            -   name: Checkout code
                uses: actions/checkout@v6

            -   name: Use Node.js
                uses: actions/setup-node@v6
                with:
                    node-version-file: '.nvmrc'
                    cache: 'npm'
                    cache-dependency-path: 'package-lock.json'

            -   name: Install Node dependencies
                run: npm ci --force --include=dev

            -   name: Build project
                run: npm run build

            -   name: Run evaluations
                run: npm run evals:run
                env:
                    GITHUB_PR_NUMBER: ${{ inputs.pr_number }}
                    PHOENIX_API_KEY: ${{ secrets.PHOENIX_API_KEY }}
                    PHOENIX_BASE_URL: ${{ secrets.PHOENIX_BASE_URL }}
                    OPENROUTER_BASE_URL: ${{ secrets.OPENROUTER_BASE_URL }}
                    OPENROUTER_API_KEY: ${{ secrets.OPENROUTER_API_KEY }}