diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 1a17b99e6a..613f7ada1f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -76,6 +76,9 @@ jobs: nightly_build_python_client: runs-on: ubuntu-latest if: github.repository == 'apache/polaris' + permissions: + # IMPORTANT: this permission is mandatory for Trusted Publishing to PyPI + id-token: write steps: - name: Checkout uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 @@ -93,8 +96,13 @@ jobs: distribution: 'temurin' java-version: '21' - - name: Publish Python client to Test PyPI - env: - UV_PUBLISH_TOKEN: ${{ secrets.TEST_PYPI_API_TOKEN }} # zizmor: ignore[secrets-outside-env] + - name: Build Python client nightly run: | - make client-nightly-publish + make client-nightly-build + + - name: Publish Python client to Test PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: client/python/dist/ + skip-existing: true diff --git a/.github/workflows/release-3-build-and-publish-artifacts.yml b/.github/workflows/release-3-build-and-publish-artifacts.yml index a7eb7fa275..63082611ac 100644 --- a/.github/workflows/release-3-build-and-publish-artifacts.yml +++ b/.github/workflows/release-3-build-and-publish-artifacts.yml @@ -523,10 +523,126 @@ jobs: | Artifact Hub metadata | ✅ Committed to dist dev | EOT + publish-python-client-rc: + name: Publish Python Client RC to PyPI + runs-on: ubuntu-latest + needs: [prerequisite-checks] + permissions: + contents: read + # IMPORTANT: this permission is mandatory for Trusted Publishing to PyPI + id-token: write + env: + DRY_RUN: ${{ needs.prerequisite-checks.outputs.dry_run }} + version_without_rc: ${{ needs.prerequisite-checks.outputs.version_without_rc }} + rc_number: ${{ needs.prerequisite-checks.outputs.rc_number }} + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up environment variables + run: | + echo "RELEASEY_DIR=$(pwd)/releasey" >> $GITHUB_ENV + echo "LIBS_DIR=$(pwd)/releasey/libs" >> $GITHUB_ENV + + - name: Set up Python + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6 + with: + python-version: "3.13" + + - name: Set up JDK for openapi-generator-cli + uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5 + with: + distribution: 'temurin' + java-version: '21' + + - name: Install Subversion + run: | + sudo apt-get update + sudo apt-get install -y subversion + + - name: Import GPG key + uses: crazy-max/ghaction-import-gpg@2dc316deee8e90f13e1a351ab510b4d5bc0c82cd # v7.0.0 + with: + gpg_private_key: ${{ secrets.POLARIS_GPG_PRIVATE_KEY }} # zizmor: ignore[secrets-outside-env] + git_user_signingkey: true + git_commit_gpgsign: true + + - name: Build Python client RC + run: | + make client-rc-build RC_VERSION="${version_without_rc}" RC_NUMBER="${rc_number}" + + - name: Sign and checksum Python distributions + run: | + source "${LIBS_DIR}/_exec.sh" + + for dist_file in client/python/dist/*.whl client/python/dist/*.tar.gz; do + calculate_sha512 "${dist_file}" + exec_process gpg --armor --output "${dist_file}.asc" --detach-sig "${dist_file}" + done + + - name: Publish Python client RC to Test PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + repository-url: https://test.pypi.org/legacy/ + packages-dir: client/python/dist/ + + - name: Stage Python wheel to Apache dist dev repository + env: + SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] + SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env] + run: | + echo "::add-mask::$SVN_PASSWORD" + + source "${LIBS_DIR}/_constants.sh" + source "${LIBS_DIR}/_exec.sh" + + dist_dev_dir=${RELEASEY_DIR}/polaris-dist-dev + + # Retry logic for SVN checkout (Apache SVN can have transient connectivity issues) + exec_process_with_retries 5 60 "${dist_dev_dir}" svn checkout --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${APACHE_DIST_URL}${APACHE_DIST_PATH}" "${dist_dev_dir}" + + python_client_version_dir="${dist_dev_dir}/python-client/${version_without_rc}" + + # If the python-client version directory already exists in SVN (e.g. re-running the workflow), + # delete it so we can re-add fresh artifacts without "already versioned" errors. + if svn info "${python_client_version_dir}" &>/dev/null 2>&1; then + exec_process svn delete "${python_client_version_dir}" + fi + + exec_process mkdir -p "${python_client_version_dir}" + exec_process cp client/python/dist/* "${python_client_version_dir}/" + + exec_process cd "${dist_dev_dir}" + exec_process svn add --parents "python-client/${version_without_rc}" + + exec_process svn commit --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive -m "Stage Apache Polaris Python client ${version_without_rc} RC${rc_number}" + + - name: Update step summary + run: | + echo "## Python Client RC" >> $GITHUB_STEP_SUMMARY + cat <> $GITHUB_STEP_SUMMARY + 🎉 Python client RC published successfully: + + | Property | Value | + | --- | --- | + | Version | \`${version_without_rc}rc${rc_number}\` | + | Registry | Test PyPI | + | Apache dist dev | \`python-client/${version_without_rc}\` | + EOT + generate-release-email: name: Generate Release Email Body runs-on: ubuntu-latest - needs: [prerequisite-checks, build-and-publish-artifacts, build-docker, build-and-stage-helm-chart] + needs: + - prerequisite-checks + - build-and-publish-artifacts + - build-docker + - build-and-stage-helm-chart + - publish-python-client-rc permissions: contents: read env: @@ -581,6 +697,12 @@ jobs: NB: you have to build the Docker images locally in order to test Helm charts. + The Python CLI RC wheel is available on: + * https://dist.apache.org/repos/dist/dev/polaris/python-client/${version_without_rc} + + The Python CLI RC is also available on Test PyPI: + * https://test.pypi.org/project/apache-polaris/${version_without_rc}rc${rc_number}/ + You can find the KEYS file here: * https://downloads.apache.org/polaris/KEYS diff --git a/.github/workflows/release-4-publish-release.yml b/.github/workflows/release-4-publish-release.yml index 9f38fea731..c80c6ec0a1 100644 --- a/.github/workflows/release-4-publish-release.yml +++ b/.github/workflows/release-4-publish-release.yml @@ -38,6 +38,8 @@ jobs: runs-on: ubuntu-latest permissions: contents: write + outputs: + version_without_rc: ${{ steps.release-params.outputs.version_without_rc }} steps: - name: Checkout repository @@ -87,6 +89,7 @@ jobs: sudo apt-get install -y subversion - name: Auto-determine release parameters from branch and Git state + id: release-params env: GIT_REF: ${{ github.ref }} run: | @@ -171,6 +174,7 @@ jobs: echo "rc_tag=${rc_tag}" >> $GITHUB_ENV echo "final_release_tag=${final_release_tag}" >> $GITHUB_ENV echo "release_branch=${current_branch}" >> $GITHUB_ENV + echo "version_without_rc=${version_without_rc}" >> $GITHUB_OUTPUT cat <> $GITHUB_STEP_SUMMARY | Parameter | Value | @@ -199,6 +203,9 @@ jobs: dev_helm_url="${APACHE_DIST_URL}/dev/polaris/helm-chart/${version_without_rc}" release_helm_url="${APACHE_DIST_URL}/release/polaris/helm-chart/${version_without_rc}" + dev_python_client_url="${APACHE_DIST_URL}/dev/polaris/python-client/${version_without_rc}" + release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client/${version_without_rc}" + exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ "${dev_artifacts_url}" "${release_artifacts_url}" \ -m "Release Apache Polaris ${version_without_rc}" @@ -207,9 +214,13 @@ jobs: "${dev_helm_url}" "${release_helm_url}" \ -m "Release Apache Polaris Helm chart ${version_without_rc}" + exec_process svn mv --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ + "${dev_python_client_url}" "${release_python_client_url}" \ + -m "Release Apache Polaris Python client ${version_without_rc}" + cat <> $GITHUB_STEP_SUMMARY ## Distribution - Artifacts and Helm chart moved from dist dev to dist release + Artifacts, Helm chart, and Python client moved from dist dev to dist release EOT - name: Clean up old releases from dist release repository @@ -265,6 +276,27 @@ jobs: echo "No old Helm chart versions found to remove" >> $GITHUB_STEP_SUMMARY fi + # List and remove old Python client versions + release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client" + old_python_client_versions=$(svn list --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive "${release_python_client_url}" | grep -E '^[0-9]+\.[0-9]+\.[0-9]+(-incubating)?/$' | sed 's|/$||' | grep -v "^${version_without_rc}$" || true) + + if [[ -n "${old_python_client_versions}" ]]; then + echo "" >> $GITHUB_STEP_SUMMARY + echo "Removing old Python client versions:" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + echo "${old_python_client_versions}" >> $GITHUB_STEP_SUMMARY + echo '```' >> $GITHUB_STEP_SUMMARY + + for old_python_client_version in ${old_python_client_versions}; do + exec_process svn rm --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ + "${release_python_client_url}/${old_python_client_version}" \ + -m "Remove old Python client ${old_python_client_version} (superseded by ${version_without_rc})" + done + else + echo "" >> $GITHUB_STEP_SUMMARY + echo "No old Python client versions found to remove" >> $GITHUB_STEP_SUMMARY + fi + - name: Transfer Helm index and artifacthub-repo.yml from dev to release env: SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] @@ -431,6 +463,13 @@ jobs: # Use the Gradle task to release the Apache staging repository with the specific staging repository ID exec_process ./gradlew releaseApacheStagingRepository --staging-repository-id "${STAGING_REPOSITORY_ID}" + cat <> $GITHUB_STEP_SUMMARY + ## Nexus + ✅ Nexus staging repository released + EOT + + - name: Release summary + run: | cat <> $GITHUB_STEP_SUMMARY ## Summary 🎉 Release published successfully: @@ -447,3 +486,58 @@ jobs: | Version | \`${version_without_rc}\` | | Final release tag | \`${final_release_tag}\` | EOT + + publish-python-client: + name: Publish Python Client to PyPI + runs-on: ubuntu-latest + needs: [publish-release] + permissions: + contents: read + # IMPORTANT: this permission is mandatory for Trusted Publishing to PyPI + id-token: write + env: + version_without_rc: ${{ needs.publish-release.outputs.version_without_rc }} + + steps: + - name: Checkout repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Set up environment variables + run: | + echo "RELEASEY_DIR=$(pwd)/releasey" >> $GITHUB_ENV + echo "LIBS_DIR=$(pwd)/releasey/libs" >> $GITHUB_ENV + + - name: Install Subversion + run: | + sudo apt-get update + sudo apt-get install -y subversion + + - name: Download Python wheel from Apache dist release repository + env: + SVN_USERNAME: ${{ secrets.POLARIS_SVN_DEV_USERNAME }} # zizmor: ignore[secrets-outside-env] + SVN_PASSWORD: ${{ secrets.POLARIS_SVN_DEV_PASSWORD }} # zizmor: ignore[secrets-outside-env] + run: | + echo "::add-mask::$SVN_PASSWORD" + + source "${LIBS_DIR}/_constants.sh" + source "${LIBS_DIR}/_exec.sh" + + release_python_client_url="${APACHE_DIST_URL}/release/polaris/python-client/${version_without_rc}" + + exec_process svn export --username "$SVN_USERNAME" --password "$SVN_PASSWORD" --non-interactive \ + "${release_python_client_url}" dist/ + + - name: Publish Python client release to PyPI + uses: pypa/gh-action-pypi-publish@ed0c53931b1dc9bd32cbe73a98c7f6766f8a527e # v1.13.0 + with: + packages-dir: dist/ + + - name: Update step summary + run: | + cat <> $GITHUB_STEP_SUMMARY + ## Python Client + ✅ Python client \`${version_without_rc}\` published to PyPI + EOT diff --git a/Makefile b/Makefile index c5902b40bf..d32b6adf35 100644 --- a/Makefile +++ b/Makefile @@ -127,18 +127,21 @@ client-install-dependencies: $(VENV_DIR) client-setup-env: $(VENV_DIR) client-install-dependencies .PHONY: client-build -client-build: client-setup-env ## Build client distribution. Pass FORMAT=sdist or FORMAT=wheel to build a specific format. +client-build: client-setup-env ## Build client distribution. Pass FORMAT=sdist or FORMAT=wheel to build a specific format, and VERSION to stamp the version before building. @echo "--- Building client distribution ---" + @if [ -n "$(VERSION)" ]; then \ + $(ACTIVATE_AND_CD) && uv version "$(VERSION)"; \ + fi @if [ -n "$(FORMAT)" ]; then \ if [ "$(FORMAT)" != "sdist" ] && [ "$(FORMAT)" != "wheel" ]; then \ echo "Error: Invalid format '$(FORMAT)'. Supported formats are 'sdist' and 'wheel'." >&2; \ exit 1; \ fi; \ echo "Building with format: $(FORMAT)"; \ - $(ACTIVATE_AND_CD) && uv build --format $(FORMAT); \ + $(ACTIVATE_AND_CD) && uv build --clear --$(FORMAT); \ else \ echo "Building default distribution (sdist and wheel)"; \ - $(ACTIVATE_AND_CD) && uv build; \ + $(ACTIVATE_AND_CD) && uv build --clear; \ fi @echo "--- Client distribution build complete ---" @@ -188,18 +191,22 @@ client-lint: client-setup-env ## Run linting checks for Polaris client @$(ACTIVATE_AND_CD) && uv run --active pre-commit run --files integration_tests/* generate_clients.py apache_polaris/cli/* apache_polaris/cli/command/* apache_polaris/cli/options/* test/* @echo "--- Client linting checks complete ---" -.PHONY: client-nightly-publish -client-nightly-publish: client-setup-env ## Build and publish nightly version to Test PyPI - @echo "--- Starting nightly publish ---" - @$(ACTIVATE_AND_CD) && \ - CURRENT_VERSION=$$(uv version --short) && \ - DATE_SUFFIX=$$(date -u +%Y%m%d%H%M%S) && \ - NIGHTLY_VERSION="$${CURRENT_VERSION}.dev$${DATE_SUFFIX}" && \ - echo "Publishing nightly version: $${NIGHTLY_VERSION}" && \ - uv version "$${NIGHTLY_VERSION}" && \ - uv build --clear && \ - uv publish --index testpypi - @echo "--- Nightly publish complete ---" +.PHONY: client-nightly-build +client-nightly-build: client-setup-env ## Build nightly version for publishing to Test PyPI + @$(MAKE) client-build \ + VERSION="$$($(VENV_DIR)/bin/uv --directory $(PYTHON_CLIENT_DIR) version --short).dev$$(date -u +%Y%m%d%H%M%S)" \ + FORMAT=wheel + +.PHONY: client-rc-build +client-rc-build: ## Build RC version for publishing to Test PyPI (requires RC_VERSION and RC_NUMBER) + @if [ -z "$(RC_VERSION)" ]; then echo "ERROR: RC_VERSION is not set"; exit 1; fi + @if [ -z "$(RC_NUMBER)" ]; then echo "ERROR: RC_NUMBER is not set"; exit 1; fi + @$(MAKE) client-build VERSION="$(RC_VERSION)rc$(RC_NUMBER)" + +.PHONY: client-release-build +client-release-build: ## Build final release version for publishing to PyPI (requires RELEASE_VERSION) + @if [ -z "$(RELEASE_VERSION)" ]; then echo "ERROR: RELEASE_VERSION is not set"; exit 1; fi + @$(MAKE) client-build VERSION="$(RELEASE_VERSION)" .PHONY: client-regenerate client-regenerate: client-setup-env ## Regenerate the client code