Merge pull request #56 from AmberLee2427/main

Add CI workflow for Nexus artefacts

Author: AmberLee2427

.github/workflows/manual-notebook-ci.yml

@@ -0,0 +1,58 @@
+name: Manual Notebook CI
+
+on:
+  workflow_dispatch:
+    inputs:
+      profile:
+        description: "Execution profile"
+        required: true
+        default: "fast"
+        type: choice
+        options:
+          - fast
+          - full
+
+jobs:
+  execute-notebooks:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Check out repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Setup micromamba
+        uses: mamba-org/setup-micromamba@v2
+        with:
+          environment-file: env.yml
+          environment-name: rges-pit-dc
+          cache-environment: true
+
+      - name: Install runner + transformer deps
+        shell: bash -l {0}
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install pyyaml nbclient nbformat
+
+      - name: Rebuild Nexus-ready notebooks (RRN/build)
+        shell: bash -l {0}
+        run: |
+          rm -rf RRN/build
+          python scripts/notebook_transformer.py --force
+
+      - name: Execute notebooks
+        shell: bash -l {0}
+        run: |
+          SKIP_TAGS=""
+          if [ "${{ inputs.profile }}" = "fast" ]; then
+            SKIP_TAGS="slow,ci-skip"
+          else
+            SKIP_TAGS="ci-skip"
+          fi
+
+          python scripts/execute_notebooks_ci.py \
+            --glob "RRN/build/*.ipynb" \
+            --skip-tags "$SKIP_TAGS" \
+            --timeout 1800

.github/workflows/publish.yml

@@ -56,9 +56,6 @@ jobs:
       contents: write
       pull-requests: write
     steps:
-      - name: Install jq
-        run: sudo apt-get update && sudo apt-get install -y jq
-
       - name: Download site bundle
         uses: actions/download-artifact@v4
         with:
@@ -68,7 +65,7 @@ jobs:
       - name: Check out site repository
         uses: actions/checkout@v4
         with:
-          repository: rges-pit/rges-pit.github.io-dc-copy
+          repository: rges-pit/rges-pit.github.io
           token: ${{ secrets.SITE_SYNC_PAT }}
           path: site
 
@@ -86,11 +83,10 @@ jobs:
           git config user.name "github-actions[bot]"
           git config user.email "github-actions[bot]@users.noreply.github.com"
 
-          # Use upstream main as base (read-only) and create/update branch on fork
-          git remote add upstream https://github.com/rges-pit/rges-pit.github.io.git
-          git fetch upstream main
-          SYNC_BRANCH="chore/data-challenge-sync-dc-copy"
-          git checkout -B "$SYNC_BRANCH" upstream/main
+          # Create/update a branch on the upstream repo
+          git fetch origin main
+          SYNC_BRANCH="chore/data-challenge-sync"
+          git checkout -B "$SYNC_BRANCH" origin/main
 
           # Stage and Commit
           git add docs/_pages/
@@ -100,59 +96,30 @@ jobs:
           fi
           git commit -m "chore: sync data-challenge workshop pages"
 
-          # Push branch to fork (dc-copy)
+          # Push branch to upstream
           git push -f origin "$SYNC_BRANCH"
 
-          # Create PR against upstream.
-          # Use REST API so we can disambiguate same-owner forks via "<fork-repo>:<branch>".
-          BASE_REPO_OWNER="rges-pit"
-          BASE_REPO_NAME="rges-pit.github.io"
-          HEAD_REPO_OWNER="rges-pit"
-          HEAD_REPO_NAME="rges-pit.github.io-dc-copy"
+          # Create PR against upstream (same repo)
+          BASE_REPO="rges-pit/rges-pit.github.io"
           HEAD_BRANCH="$SYNC_BRANCH"
           BASE_BRANCH="main"
           PR_TITLE="Draft: Sync data-challenge notebooks"
           PR_BODY="Automated sync of generated workshop documentation from \`data-challenge-notebooks\`."
 
-          # Find an existing open PR from this fork branch.
-          PULLS_JSON=$(gh api \
-            --paginate \
-            -H "Accept: application/vnd.github+json" \
-            "/repos/$BASE_REPO_OWNER/$BASE_REPO_NAME/pulls?state=open&per_page=100")
-
-          EXISTING_PR=$(echo "$PULLS_JSON" | jq -r '.[] | select(.head.ref=="'"$HEAD_BRANCH"'" and .head.repo.name=="'"$HEAD_REPO_NAME"'") | .html_url' | head -n 1)
-          if [ -n "$EXISTING_PR" ] && [ "$EXISTING_PR" != "null" ]; then
+          EXISTING_PR=$(gh pr list --repo "$BASE_REPO" --head "$HEAD_BRANCH" --state open --json url --jq '.[0].url')
+          if [ -n "$EXISTING_PR" ]; then
             echo "PR already exists: $EXISTING_PR"
             exit 0
           fi
 
           echo "Creating new PR..."
-          set +e
-          CREATE_PR_JSON=$(gh api \
-            -H "Accept: application/vnd.github+json" \
-            -X POST \
-            "/repos/$BASE_REPO_OWNER/$BASE_REPO_NAME/pulls" \
-            -f title="$PR_TITLE" \
-            -f body="$PR_BODY" \
-            -f head="$HEAD_REPO_OWNER:$HEAD_BRANCH" \
-            -f base="$BASE_BRANCH" \
-            -F draft=true 2>&1)
-          CREATE_STATUS=$?
-          set -e
-
-          if [ $CREATE_STATUS -ne 0 ]; then
-            echo "PR creation failed (exit $CREATE_STATUS). Full response:"
-            echo "$CREATE_PR_JSON"
-            exit 1
-          fi
-
-          NEW_PR_URL=$(echo "$CREATE_PR_JSON" | jq -r '.html_url')
-          if [ -z "$NEW_PR_URL" ] || [ "$NEW_PR_URL" = "null" ]; then
-            echo "PR creation did not return a URL. Full response:"
-            echo "$CREATE_PR_JSON" | jq -r '.'
-            exit 1
-          fi
-          echo "Created PR: $NEW_PR_URL"
+          gh pr create \
+            --repo "$BASE_REPO" \
+            --head "$HEAD_BRANCH" \
+            --base "$BASE_BRANCH" \
+            --title "$PR_TITLE" \
+            --body "$PR_BODY" \
+            --draft
 
   nexus-publish:
     needs: build-pages

README.md

@@ -92,13 +92,14 @@ Before submitting a pull request, please check whether an existing issue or disc
 
 ## Syncing Notebooks to the Roman Research Nexus
 
-Syncing notebooks from canonical sources in this repo with the STScI `roman_notebooks` repository is an automated process handled by GitHub Actions when a pull request is merged.
+Syncing Nexus-ready notebook artifacts is an automated process handled by GitHub Actions when a pull request is merged.
 
 The workflow will:
-- Copy the notebook to the appropriate location in the `roman_notebooks` repository.
-- Validate that relative paths in the notebook are correct for the RRN structure.
-- Correct style and contents to satisfy Nexus-specific requirements.
-- Copy any associated `requirements.txt` or `env.yml` files.
+- Rebuild Nexus-ready notebook artifacts into `RRN/build/` using `scripts/notebook_transformer.py` and `notebooks_manifest.yml`.
+- Export those artifacts into the dedicated `rges-pit/nexus-notebooks` repository via the `RRN/export_submodule/` submodule.
+- Force-push the artifact branch (`main`) because the destination repository is treated as export-only.
+
+For a manual, on-demand execution check of the build artifacts, see `RRN/README.md` (Manual Notebook CI).
 
 ## Notes on Using and Contributing from Colab
 

RRN/README.md

@@ -9,3 +9,19 @@
 Notebooks in this directory are managed by GitHub workflow `.github/workflows/publish.yml`. 
 
 See notebooks_manifest for canonical source file: `source_path:`.
+
+### Manual Notebook CI (Build Artifacts)
+
+This repository includes a manual (workflow-dispatch) CI job that rebuilds the
+Nexus-ready notebook artifacts in `RRN/build/` and then executes them.
+
+- Workflow: `.github/workflows/manual-notebook-ci.yml`
+- Profiles:
+    - **fast**: skips cells tagged `slow` or `ci-skip`
+    - **full**: skips only `ci-skip`
+
+Notes:
+- The workflow deletes `RRN/build/` before rebuilding to avoid executing stale
+    notebooks (for example, notebooks marked `nexus_support: false` in the
+    manifest).
+- Cell tags can be edited in the notebook UI (cell metadata tags).

RRN/export_submodule

@@ -28,11 +28,13 @@
   - `reges-pit/nexus-notebooks`
 - [ ] add global `refdata_dependencies.yaml` (match `spacetelescope/roman_notebooks` format)
 - [ ] decide notebook discovery regexes (paths/substrings STScI wrangler should scan)
-- [ ] organize notebooks in directories where each dir has its own `requirements.txt`
-- [ ] add optional top-level `environment.yml` (mamba spec for non-pip / low-level deps only)
-- [ ] move any per-notebook helper `.py` into the same notebook directories
-- [ ] if using shared modules, create a top-level shared dir + symlink into each notebook dir that imports it
-- [ ] update automation: publish/update the Nexus repo from `RRN/export/` (no PRs to `roman_notebooks`)
+  - `notebooks/*.ipynb`
+- [-] organize notebooks in directories where each dir has its own `requirements.txt`
+- [x] add optional top-level `environment.yml` (mamba spec for non-pip / low-level deps only)
+- [-] move any per-notebook helper `.py` into the same notebook directories
+   - they are downloaded in cells
+- [-] if using shared modules, create a top-level shared dir + symlink into each notebook dir that imports it
+- [x] update automation: publish/update the Nexus repo from `RRN/export_submodule/` (no PRs to `roman_notebooks`)
 - [ ] send STScI the repo URL + chosen regex list + any notes on expected execution order
 
 ## Nexus
@@ -44,6 +46,8 @@
 - [-] edit the tools notebook to match the Colab/RRN freindly style
 - [ ] add Citations sections in all notebooks (is this already done?)
 - [ ] delete the `roman_notebooks` PR + fork
+- [ ] test that the notebooks run using the provided yaml
+- [ ] tag long cells to avoid in quick tests
 
 ## Microlensing Tools Notebook
 - [ ] table formatting
@@ -59,4 +63,10 @@
 - [ ] proof read
 - [ ] test run
 - [ ] check data import functions
-- [ ] Add to nexus export list
+- [ ] Add to nexus export list
+
+## Website
+- [ ] remove mention of workshop sign-up
+- [ ] update with real data links
+- [x] change PR from fork to PR from branch
+- [ ] delete website fork (dc-copy)

TODO.md

@@ -75,6 +75,8 @@ def export_rrn_notebooks(
     # for requirements.txt placement).
     notebook_dirs: set[Path] = set()
     for entry in notebooks:
+        if not entry.get("nexus_support", False):
+            continue
         target = entry.get("rrn_target")
         nb_id = entry.get("id")
         if not nb_id or not target:
@@ -89,6 +91,8 @@ def export_rrn_notebooks(
                 shutil.rmtree(nb_dir)
 
     for entry in notebooks:
+        if not entry.get("nexus_support", False):
+            continue
         nb_id = entry.get("id")
         target = entry.get("rrn_target")
         if not nb_id or not target: