feat: BYOK and backend cost tracking

.github/workflows/_build-backend.yml

@@ -0,0 +1,97 @@
+name: Build and deploy backend
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string # dev, stg, prd
+      ref:
+        required: true
+        type: string # development, staging, main
+      version_tag:
+        required: true
+        type: string # e.g. "development" or "v2.1.3"
+
+jobs:
+  build-backend-and-push:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+    outputs:
+      IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }}
+      PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+      - name: Setup env
+        run: |
+          echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV
+          echo "BRANCH_NAME=${{ inputs.version_tag }}" >> $GITHUB_ENV
+          echo "VERSION=${{ inputs.version_tag }}" >> $GITHUB_ENV
+      - name: Log in to the container registry
+        uses: docker/login-action@v3
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Build and push docker image
+        run: |
+          make all
+      - id: output
+        name: Output image tag
+        run: |
+          export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION}
+          echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT
+          echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT
+
+  deploy-backend:
+    needs: build-backend-and-push
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+      - name: Clone deploy repo
+        run: |
+          git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy
+      - name: Generate kubernetes manifests
+        env:
+          OPENAI_BASE_URL: ${{ inputs.environment == 'dev' && secrets.OPENAI_BASE_URL_DEV || inputs.environment == 'stg' && secrets.OPENAI_BASE_URL_STG || secrets.OPENAI_BASE_URL_PRD }}
+          OPENAI_API_KEY: ${{ inputs.environment == 'dev' && secrets.OPENAI_API_KEY_DEV || inputs.environment == 'stg' && secrets.OPENAI_API_KEY_STG || secrets.OPENAI_API_KEY_PRD }}
+          INFERENCE_BASE_URL: ${{ inputs.environment == 'dev' && secrets.INFERENCE_BASE_URL_DEV || inputs.environment == 'stg' && secrets.INFERENCE_BASE_URL_STG || secrets.INFERENCE_BASE_URL_PRD }}
+          INFERENCE_API_KEY: ${{ inputs.environment == 'dev' && secrets.INFERENCE_API_KEY_DEV || inputs.environment == 'stg' && secrets.INFERENCE_API_KEY_STG || secrets.INFERENCE_API_KEY_PRD }}
+          MCP_BASIC_KEY: ${{ inputs.environment == 'dev' && secrets.MCP_BASIC_KEY_DEV || inputs.environment == 'stg' && secrets.MCP_BASIC_KEY_STG || secrets.MCP_BASIC_KEY_PRD }}
+          MCP_PAPERSCORE_KEY: ${{ inputs.environment == 'dev' && secrets.MCP_PAPERSCORE_KEY_DEV || inputs.environment == 'stg' && secrets.MCP_PAPERSCORE_KEY_STG || secrets.MCP_PAPERSCORE_KEY_PRD }}
+          XTRAMCP_OPENAI_BASE_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENAI_BASE_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENAI_BASE_URL_STG || secrets.XTRAMCP_OPENAI_BASE_URL_PRD }}
+          XTRAMCP_OPENAI_API_KEY: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENAI_API_KEY_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENAI_API_KEY_STG || secrets.XTRAMCP_OPENAI_API_KEY_PRD }}
+          XTRAMCP_OPENREVIEW_BASE_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENREVIEW_BASE_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENREVIEW_BASE_URL_STG || secrets.XTRAMCP_OPENREVIEW_BASE_URL_PRD }}
+          XTRAMCP_OPENREVIEW_USERNAME: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENREVIEW_USERNAME_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENREVIEW_USERNAME_STG || secrets.XTRAMCP_OPENREVIEW_USERNAME_PRD }}
+          XTRAMCP_OPENREVIEW_PASSWORD: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_OPENREVIEW_PASSWORD_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_OPENREVIEW_PASSWORD_STG || secrets.XTRAMCP_OPENREVIEW_PASSWORD_PRD }}
+          XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_STG || secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_PRD }}
+          XTRAMCP_DOI_EMAIL_ADDRESS: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_DOI_EMAIL_ADDRESS_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_DOI_EMAIL_ADDRESS_STG || secrets.XTRAMCP_DOI_EMAIL_ADDRESS_PRD }}
+          XTRAMCP_ACL_METADATA_DB_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_ACL_METADATA_DB_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_ACL_METADATA_DB_URL_STG || secrets.XTRAMCP_ACL_METADATA_DB_URL_PRD }}
+          XTRAMCP_ARXIV_METADATA_DB_URL: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_ARXIV_METADATA_DB_URL_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_ARXIV_METADATA_DB_URL_STG || secrets.XTRAMCP_ARXIV_METADATA_DB_URL_PRD }}
+          XTRAMCP_MONGO_URI: ${{ inputs.environment == 'dev' && secrets.XTRAMCP_MONGO_URI_DEV || inputs.environment == 'stg' && secrets.XTRAMCP_MONGO_URI_STG || secrets.XTRAMCP_MONGO_URI_PRD }}
+          MONGO_URI: ${{ inputs.environment == 'dev' && secrets.MONGO_URI_DEV || inputs.environment == 'stg' && secrets.MONGO_URI_STG || secrets.MONGO_URI_PRD }}
+          GHCR_DOCKER_CONFIG: ${{ inputs.environment == 'dev' && secrets.GHCR_DOCKER_CONFIG_DEV || inputs.environment == 'stg' && secrets.GHCR_DOCKER_CONFIG_STG || secrets.GHCR_DOCKER_CONFIG_PRD }}
+          CLOUDFLARE_TUNNEL_TOKEN: ${{ inputs.environment == 'dev' && secrets.CLOUDFLARE_TUNNEL_TOKEN_DEV || inputs.environment == 'stg' && secrets.CLOUDFLARE_TUNNEL_TOKEN_STG || secrets.CLOUDFLARE_TUNNEL_TOKEN_PRD }}
+        run: |
+          export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }}
+          mkdir -p ../deploy/${{ inputs.environment }}
+          ./hack/${{ inputs.environment }}.sh > ../deploy/${{ inputs.environment }}/paperdebugger.yaml
+      - name: Push changes to deploy repo
+        run: |
+          export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }}
+          cd ../deploy
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add ${{ inputs.environment }}/paperdebugger.yaml
+          git diff --staged --quiet || git commit -m "chore: update paperdebugger ${{ inputs.environment }}, revision ${IMAGE_TAG}"
+          git push

.github/workflows/_build-ext.yml

@@ -0,0 +1,70 @@
+name: Build and publish Chrome extension
+
+on:
+  workflow_call:
+    inputs:
+      environment:
+        required: true
+        type: string # stg, prd
+      ref:
+        required: true
+        type: string # staging, main
+      extension_id:
+        required: true
+        type: string
+
+jobs:
+  build-chrome-extension:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.ref }}
+          fetch-depth: 0
+
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: lts/*
+
+      - name: Build
+        run: |
+          export GRAFANA_API_KEY=${{ secrets.GRAFANA_API_KEY }}
+          cd webapp/_webapp
+          npm install
+          npm run build:${{ inputs.environment }}:chrome
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: paperdebugger-chrome-extension-${{ inputs.environment }}
+          path: webapp/_webapp/dist
+
+  push-to-chrome-web-store:
+    runs-on: ubuntu-latest
+    needs: build-chrome-extension
+    steps:
+      - name: Install zip
+        run: |
+          sudo apt-get update && sudo apt-get install -y zip
+
+      - name: Download artifacts
+        uses: actions/download-artifact@v4
+        with:
+          name: paperdebugger-chrome-extension-${{ inputs.environment }}
+          path: dist
+
+      - name: Zip extension files
+        run: |
+          zip -r dist.zip dist/*
+
+      - name: Upload to Chrome Web Store (upload only)
+        uses: mobilefirstllc/cws-publish@latest
+        with:
+          action: 'upload'
+          client_id: ${{ secrets.CHROME_EXT_CLIENT_ID }}
+          client_secret: ${{ secrets.CHROME_EXT_CLIENT_SECRET }}
+          refresh_token: ${{ secrets.CHROME_EXT_REFRESH_TOKEN }}
+          extension_id: ${{ inputs.extension_id }}
+          zip_file: 'dist.zip'

.github/workflows/build-backend-dev.yml

@@ -4,84 +4,15 @@ on:
     branches:
       - development
 
-jobs:
-  build-backend-and-push:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    outputs:
-      IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }}
-      PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: development
-      - name: Setup env
-        run: |
-          echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV
-          echo "BRANCH_NAME=development" >> $GITHUB_ENV
-          echo "VERSION=development" >> $GITHUB_ENV
-      - name: Log in to the container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push docker image
-        run: |
-          make all
-      - id: output
-        name: Output image tag
-        run: |
-          export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION}
-          echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT
-          echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT
+permissions:
+  contents: write
+  packages: write
 
-  deploy-backend:
-    needs: build-backend-and-push
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: development
-      - name: Clone deploy repo
-        run: |
-          git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy
-      - name: Generate kubernetes manifests
-        env:
-          OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL_DEV }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_DEV }}
-          INFERENCE_BASE_URL: ${{ secrets.INFERENCE_BASE_URL_DEV }}
-          INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY_DEV }}
-          MCP_BASIC_KEY: ${{ secrets.MCP_BASIC_KEY_DEV }}
-          MCP_PAPERSCORE_KEY: ${{ secrets.MCP_PAPERSCORE_KEY_DEV }}
-          XTRAMCP_OPENAI_BASE_URL: ${{ secrets.XTRAMCP_OPENAI_BASE_URL_DEV }}
-          XTRAMCP_OPENAI_API_KEY: ${{ secrets.XTRAMCP_OPENAI_API_KEY_DEV }}
-          XTRAMCP_OPENREVIEW_BASE_URL: ${{ secrets.XTRAMCP_OPENREVIEW_BASE_URL_DEV }}
-          XTRAMCP_OPENREVIEW_USERNAME: ${{ secrets.XTRAMCP_OPENREVIEW_USERNAME_DEV }}
-          XTRAMCP_OPENREVIEW_PASSWORD: ${{ secrets.XTRAMCP_OPENREVIEW_PASSWORD_DEV }}
-          XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_DEV }}
-          XTRAMCP_DOI_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_DOI_EMAIL_ADDRESS_DEV }}
-          XTRAMCP_ARXIV_METADATA_DB_URL: ${{ secrets.XTRAMCP_ARXIV_METADATA_DB_URL_DEV }}
-          XTRAMCP_MONGO_URI: ${{ secrets.XTRAMCP_MONGO_URI_DEV }}
-          MONGO_URI: ${{ secrets.MONGO_URI_DEV }}
-          GHCR_DOCKER_CONFIG: ${{ secrets.GHCR_DOCKER_CONFIG_DEV }}
-          CLOUDFLARE_TUNNEL_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_DEV }}
-        run: |
-          export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }}
-          mkdir -p ../deploy/dev
-          ./hack/dev.sh > ../deploy/dev/paperdebugger.yaml
-      - name: Push changes to deploy repo
-        run: |
-          export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }}
-          cd ../deploy
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git add dev/paperdebugger.yaml
-          git diff --staged --quiet || git commit -m "chore: update paperdebugger dev, revision ${IMAGE_TAG}"
-          git push
+jobs:
+  build-and-deploy:
+    uses: ./.github/workflows/_build-backend.yml
+    secrets: inherit
+    with:
+      environment: dev
+      ref: development
+      version_tag: development

.github/workflows/build-backend-prd.yml

@@ -1,90 +1,17 @@
 name: build backend for prd
 on:
-  push:
-    tags:
-      - 'v*'
   repository_dispatch:
     types: [prd]
 
-jobs:
-  build-backend-and-push:
-    runs-on: ubuntu-latest
-    permissions:
-      contents: read
-      packages: write
-    outputs:
-      IMAGE_TAG: ${{ steps.output.outputs.IMAGE_TAG }}
-      PAPERDEBUGGER_IMAGE: ${{ steps.output.outputs.PAPERDEBUGGER_IMAGE }}
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: main
-      - name: Setup env
-        run: |
-          echo "MONOREPO_REVISION=$(git rev-parse HEAD | cut -c1-6)" >> $GITHUB_ENV
-          echo "BRANCH_NAME=${{ github.event.client_payload.tag }}" >> $GITHUB_ENV
-          echo "VERSION=${{ github.event.client_payload.tag }}" >> $GITHUB_ENV
-      - name: Log in to the container registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.actor }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Build and push docker image
-        run: |
-          make all
-      - id: output
-        name: Output image tag
-        run: |
-          export IMAGE_TAG=${BRANCH_NAME}-${MONOREPO_REVISION}
-          echo "IMAGE_TAG=${IMAGE_TAG}" >> $GITHUB_OUTPUT
-          echo "PAPERDEBUGGER_IMAGE=ghcr.io/paperdebugger/sharelatex-paperdebugger:${IMAGE_TAG}" >> $GITHUB_OUTPUT
+permissions:
+  contents: write
+  packages: write
 
-  deploy-backend:
-    needs: build-backend-and-push
-    runs-on: ubuntu-latest
-    permissions:
-      contents: write
-    steps:
-      - name: Checkout repository
-        uses: actions/checkout@v4
-        with:
-          ref: main
-      - name: Clone deploy repo
-        run: |
-          git clone https://${{ secrets.GH_PAT }}@github.com/paperdebugger/deploy.git ../deploy
-      - name: Generate kubernetes manifests
-        env:
-          OPENAI_BASE_URL: ${{ secrets.OPENAI_BASE_URL_PRD }}
-          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY_PRD }}
-          INFERENCE_BASE_URL: ${{ secrets.INFERENCE_BASE_URL_PRD }}
-          INFERENCE_API_KEY: ${{ secrets.INFERENCE_API_KEY_PRD }}
-          MCP_BASIC_KEY: ${{ secrets.MCP_BASIC_KEY_PRD }}
-          MCP_PAPERSCORE_KEY: ${{ secrets.MCP_PAPERSCORE_KEY_PRD }}
-          XTRAMCP_OPENAI_BASE_URL: ${{ secrets.XTRAMCP_OPENAI_BASE_URL_PRD }}
-          XTRAMCP_OPENAI_API_KEY: ${{ secrets.XTRAMCP_OPENAI_API_KEY_PRD }}
-          XTRAMCP_OPENREVIEW_BASE_URL: ${{ secrets.XTRAMCP_OPENREVIEW_BASE_URL_PRD }}
-          XTRAMCP_OPENREVIEW_USERNAME: ${{ secrets.XTRAMCP_OPENREVIEW_USERNAME_PRD }}
-          XTRAMCP_OPENREVIEW_PASSWORD: ${{ secrets.XTRAMCP_OPENREVIEW_PASSWORD_PRD }}
-          XTRAMCP_CROSSREF_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_CROSSREF_EMAIL_ADDRESS_PRD }}
-          XTRAMCP_DOI_EMAIL_ADDRESS: ${{ secrets.XTRAMCP_DOI_EMAIL_ADDRESS_PRD }}
-          XTRAMCP_ACL_METADATA_DB_URL: ${{ secrets.XTRAMCP_ACL_METADATA_DB_URL_PRD }}
-          XTRAMCP_ARXIV_METADATA_DB_URL: ${{ secrets.XTRAMCP_ARXIV_METADATA_DB_URL_PRD }}
-          XTRAMCP_MONGO_URI: ${{ secrets.XTRAMCP_MONGO_URI_PRD }}
-          MONGO_URI: ${{ secrets.MONGO_URI_PRD }}
-          GHCR_DOCKER_CONFIG: ${{ secrets.GHCR_DOCKER_CONFIG_PRD }}
-          CLOUDFLARE_TUNNEL_TOKEN: ${{ secrets.CLOUDFLARE_TUNNEL_TOKEN_PRD }}
-        run: |
-          export PAPERDEBUGGER_IMAGE=${{ needs.build-backend-and-push.outputs.PAPERDEBUGGER_IMAGE }}
-          mkdir -p ../deploy/prd
-          ./hack/prd.sh > ../deploy/prd/paperdebugger.yaml
-      - name: Push changes to deploy repo
-        run: |
-          export IMAGE_TAG=${{ needs.build-backend-and-push.outputs.IMAGE_TAG }}
-          cd ../deploy
-          git config --global user.name "github-actions[bot]"
-          git config --global user.email "github-actions[bot]@users.noreply.github.com"
-          git add prd/paperdebugger.yaml
-          git diff --staged --quiet || git commit -m "chore: update paperdebugger prd, revision ${IMAGE_TAG}"
-          git push
+jobs:
+  build-and-deploy:
+    uses: ./.github/workflows/_build-backend.yml
+    secrets: inherit
+    with:
+      environment: prd
+      ref: main
+      version_tag: ${{ github.event.client_payload.tag }}