From 5ff9142a94fb96d289ec560bef13a74bfae96d21 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:52:12 +0000 Subject: [PATCH 01/10] Move CI to Databricks protected runners with JFrog OIDC - Add .github/actions/setup-jfrog composite action for OIDC-based JFrog authentication, configuring both pip and Poetry - Switch all workflow jobs from ubuntu-latest to databricks-protected-runner-group - Replace third-party tisonkun/actions-dco with inline bash script for DCO sign-off checking - Update actions/checkout to v4 and actions/setup-python to v5 - Add id-token: write permission for JFrog OIDC token exchange Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-jfrog/action.yml | 39 ++++++++++ .github/workflows/code-quality-checks.yml | 23 ++++-- .github/workflows/dco-check.yml | 91 +++++++++++++++++------ .github/workflows/integration.yml | 11 ++- 4 files changed, 131 insertions(+), 33 deletions(-) create mode 100644 .github/actions/setup-jfrog/action.yml diff --git a/.github/actions/setup-jfrog/action.yml b/.github/actions/setup-jfrog/action.yml new file mode 100644 index 0000000..04cbfcf --- /dev/null +++ b/.github/actions/setup-jfrog/action.yml @@ -0,0 +1,39 @@ +name: Setup JFrog OIDC +description: Obtain a JFrog access token via GitHub OIDC and configure pip/Poetry to use JFrog PyPI proxy + +runs: + using: composite + steps: + - name: Get JFrog OIDC token + shell: bash + run: | + set -euo pipefail + ID_TOKEN=$(curl -sLS \ + -H "User-Agent: actions/oidc-client" \ + -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \ + "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"') + echo "::add-mask::${ID_TOKEN}" + ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \ + "https://databricks.jfrog.io/access/api/v1/oidc/token" \ + -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"') + echo "::add-mask::${ACCESS_TOKEN}" + if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then + echo "FAIL: Could not extract JFrog access token" + exit 1 + fi + echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" + echo "JFrog OIDC token obtained successfully" + + - name: Configure pip and Poetry + shell: bash + run: | + set -euo pipefail + + # Configure pip index (used by Poetry's installer under the hood) + echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" + + # Configure JFrog as a supplemental source for Poetry + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + + echo "pip and Poetry configured to use JFrog registry" diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 923a3c1..36712fd 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -10,10 +10,13 @@ on: permissions: contents: read + id-token: write jobs: check-linting: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest strategy: matrix: python-version: [3.9, "3.10", "3.11", "3.12"] @@ -22,10 +25,12 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- @@ -66,7 +71,9 @@ jobs: run: poetry run black --check src check-types: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest strategy: matrix: python-version: [3.9, "3.10", "3.11", "3.12"] @@ -75,10 +82,12 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python - uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} #---------------------------------------------- @@ -118,4 +127,4 @@ jobs: - name: Mypy run: | mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 - poetry run mypy --install-types --non-interactive src \ No newline at end of file + poetry run mypy --install-types --non-interactive src diff --git a/.github/workflows/dco-check.yml b/.github/workflows/dco-check.yml index 5cb19c2..fdcf1b3 100644 --- a/.github/workflows/dco-check.yml +++ b/.github/workflows/dco-check.yml @@ -1,29 +1,74 @@ name: DCO Check -on: [pull_request] +on: + pull_request: + types: [opened, synchronize, reopened] + branches: [main] permissions: - contents: read - pull-requests: write + contents: read jobs: - check: - runs-on: ubuntu-latest - steps: - - name: Check for DCO - id: dco-check - uses: tisonkun/actions-dco@6d1f8a197db1b04df1769707b46b9366b1eca902 # v1.1 - - name: Comment about DCO status - uses: actions/github-script@00f12e3e20659f42342b1c0226afda7f7c042325 # v6 - if: ${{ failure() }} - with: - script: | - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: `Thanks for your contribution! To satisfy the DCO policy in our \ - [contributing guide](https://github.com/databricks/databricks-sqlalchemy/blob/main/CONTRIBUTING.md) \ - every commit message must include a sign-off message. One or more of your commits is missing this message. \ - You can reword previous commit messages with an interactive rebase (\`git rebase -i main\`).` - }) \ No newline at end of file + dco-check: + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest + name: Check DCO Sign-off + steps: + - name: Checkout + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + with: + fetch-depth: 0 + + - name: Check DCO Sign-off + env: + BASE_SHA: ${{ github.event.pull_request.base.sha }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + run: | + #!/bin/bash + set -e + + echo "Checking commits from $BASE_SHA to $HEAD_SHA" + + COMMITS=$(git rev-list --no-merges "$BASE_SHA..$HEAD_SHA") + + if [ -z "$COMMITS" ]; then + echo "No commits found in this PR" + exit 0 + fi + + FAILED_COMMITS=() + + for commit in $COMMITS; do + echo "Checking commit: $commit" + COMMIT_MSG=$(git log --format=%B -n 1 "$commit") + if echo "$COMMIT_MSG" | grep -q "^Signed-off-by: "; then + echo " Commit $commit has DCO sign-off" + else + echo " Commit $commit is missing DCO sign-off" + FAILED_COMMITS+=("$commit") + fi + done + + if [ ${#FAILED_COMMITS[@]} -ne 0 ]; then + echo "" + echo "DCO Check Failed!" + echo "The following commits are missing the required 'Signed-off-by' line:" + for commit in "${FAILED_COMMITS[@]}"; do + echo " - $commit: $(git log --format=%s -n 1 "$commit")" + done + echo "" + echo "To fix this, you need to sign off your commits. You can:" + echo "1. Add sign-off to new commits: git commit -s -m 'Your commit message'" + echo "2. Amend existing commits: git commit --amend --signoff" + echo "3. For multiple commits, use: git rebase --signoff HEAD~N (where N is the number of commits)" + echo "" + echo "The sign-off should be in the format:" + echo "Signed-off-by: Your Name " + echo "" + echo "For more details, see CONTRIBUTING.md" + exit 1 + else + echo "" + echo "All commits have proper DCO sign-off!" + fi diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 3f7d2d9..c7528d0 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -9,10 +9,13 @@ on: permissions: contents: read + id-token: write jobs: run-e2e-tests: - runs-on: ubuntu-latest + runs-on: + group: databricks-protected-runner-group + labels: linux-ubuntu-latest environment: azure-prod env: DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} @@ -26,10 +29,12 @@ jobs: # check-out repo and set-up python #---------------------------------------------- - name: Check out repository - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3 + uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python - uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4 + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" #---------------------------------------------- From bf9644a84613cbb6ec619bd080a4d0bee58a400b Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:53:48 +0000 Subject: [PATCH 02/10] Fix setup-jfrog: remove poetry config calls that run before poetry install The composite action runs before Poetry is installed, so poetry commands fail with "command not found". Keep only PIP_INDEX_URL which Poetry's pip-based installer respects (matching databricks-sql-python2 pattern). Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-jfrog/action.yml | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/.github/actions/setup-jfrog/action.yml b/.github/actions/setup-jfrog/action.yml index 04cbfcf..97ae146 100644 --- a/.github/actions/setup-jfrog/action.yml +++ b/.github/actions/setup-jfrog/action.yml @@ -1,5 +1,5 @@ name: Setup JFrog OIDC -description: Obtain a JFrog access token via GitHub OIDC and configure pip/Poetry to use JFrog PyPI proxy +description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy runs: using: composite @@ -24,16 +24,9 @@ runs: echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV" echo "JFrog OIDC token obtained successfully" - - name: Configure pip and Poetry + - name: Configure pip shell: bash run: | set -euo pipefail - - # Configure pip index (used by Poetry's installer under the hood) echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV" - - # Configure JFrog as a supplemental source for Poetry - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - - echo "pip and Poetry configured to use JFrog registry" + echo "pip configured to use JFrog registry" From 6b0cb52cde9d78718f9923e15c4509360802782f Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:55:58 +0000 Subject: [PATCH 03/10] Move JFrog setup after Poetry install to avoid blocking installer The snok/install-poetry action downloads from install.python-poetry.org which is blocked by the hardened runner firewall. Moving JFrog setup (which sets PIP_INDEX_URL) to after Poetry is installed ensures the installer can reach its metadata endpoint, while dependencies still resolve through JFrog. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 14 ++++++++++---- .github/workflows/integration.yml | 7 +++++-- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 36712fd..16655ca 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -26,8 +26,6 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -43,6 +41,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # setup jfrog for dependency install + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog #---------------------------------------------- # load cached venv if cache exists @@ -83,8 +86,6 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -100,6 +101,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # setup jfrog for dependency install + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index c7528d0..35cf820 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -30,8 +30,6 @@ jobs: #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 @@ -47,6 +45,11 @@ jobs: virtualenvs-create: true virtualenvs-in-project: true installer-parallel: true + #---------------------------------------------- + # setup jfrog for dependency install + #---------------------------------------------- + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog #---------------------------------------------- # load cached venv if cache exists From 2a178c0f0cf6f81665f76cd6a71f74863a163afd Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 09:57:47 +0000 Subject: [PATCH 04/10] Replace snok/install-poetry with pip install from JFrog The snok/install-poetry action downloads from install.python-poetry.org which is blocked by the hardened runner firewall. Install Poetry via pip from JFrog instead, and configure virtualenv settings manually. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 38 +++++++++++------------ .github/workflows/integration.yml | 19 ++++++------ 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 16655ca..f42439f 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -32,20 +32,19 @@ jobs: with: python-version: ${{ matrix.python-version }} #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # setup jfrog for dependency install #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true #---------------------------------------------- # load cached venv if cache exists @@ -92,20 +91,19 @@ jobs: with: python-version: ${{ matrix.python-version }} #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # setup jfrog for dependency install #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true #---------------------------------------------- # load cached venv if cache exists diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 35cf820..35b9517 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -36,20 +36,19 @@ jobs: with: python-version: "3.10" #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 - with: - version: "2.2.1" - virtualenvs-create: true - virtualenvs-in-project: true - installer-parallel: true - #---------------------------------------------- # setup jfrog for dependency install #---------------------------------------------- - name: Setup JFrog uses: ./.github/actions/setup-jfrog + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true #---------------------------------------------- # load cached venv if cache exists From c70a711518422d0e4caab50ad56407047e36dcf9 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:00:26 +0000 Subject: [PATCH 05/10] Move JFrog setup before setup-python to fix pip upgrade on uncached versions actions/setup-python upgrades pip when installing a Python version not in the runner cache (e.g. 3.9). This hits pypi.org which is blocked on the hardened runners. Moving JFrog setup before setup-python ensures PIP_INDEX_URL is set so pip upgrades resolve through JFrog. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 54 +++-------------------- .github/workflows/integration.yml | 23 +--------- 2 files changed, 7 insertions(+), 70 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index f42439f..fc3fcf3 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -21,54 +21,32 @@ jobs: matrix: python-version: [3.9, "3.10", "3.11", "3.12"] steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # setup jfrog for dependency install - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - name: Install library run: poetry install --no-interaction - #---------------------------------------------- - # black the code - #---------------------------------------------- - name: Black run: poetry run black --check src @@ -80,55 +58,33 @@ jobs: matrix: python-version: [3.9, "3.10", "3.11", "3.12"] steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python ${{ matrix.python-version }} id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: ${{ matrix.python-version }} - #---------------------------------------------- - # setup jfrog for dependency install - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' run: poetry install --no-interaction --no-root - #---------------------------------------------- - # install your root project, if required - #---------------------------------------------- - name: Install library run: poetry install --no-interaction - #---------------------------------------------- - # mypy the code - #---------------------------------------------- - name: Mypy run: | - mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 + mkdir .mypy_cache poetry run mypy --install-types --non-interactive src diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 35b9517..18c6a63 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -25,47 +25,28 @@ jobs: DATABRICKS_SCHEMA: ${{ secrets.SQLALCHEMY_SCHEMA }} DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} steps: - #---------------------------------------------- - # check-out repo and set-up python - #---------------------------------------------- - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog - name: Set up python id: setup-python uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 with: python-version: "3.10" - #---------------------------------------------- - # setup jfrog for dependency install - #---------------------------------------------- - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - #---------------------------------------------- - # ----- install & configure poetry ----- - #---------------------------------------------- - name: Install Poetry run: | pip install poetry==2.2.1 poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - - #---------------------------------------------- - # load cached venv if cache exists - #---------------------------------------------- - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 with: path: .venv key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - #---------------------------------------------- - # install dependencies if cache does not exist - #---------------------------------------------- - name: Install dependencies run: poetry install --no-interaction --all-extras - #---------------------------------------------- - # run test suite - #---------------------------------------------- - name: Run SQL Alchemy tests run: poetry run python -m pytest tests/test_local From 98987e1332687ab2eb03d04288de738a2d4a645a Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:02:37 +0000 Subject: [PATCH 06/10] Add Poetry JFrog source config for dependency resolution Poetry does not use PIP_INDEX_URL for dependency resolution. Configure JFrog as the primary Poetry source via poetry source add so that poetry install resolves packages through JFrog instead of pypi.org. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 10 ++++++++++ .github/workflows/integration.yml | 5 +++++ 2 files changed, 15 insertions(+) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index fc3fcf3..b532615 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -36,6 +36,11 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true + - name: Configure Poetry JFrog source + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 @@ -73,6 +78,11 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true + - name: Configure Poetry JFrog source + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 18c6a63..9201c9f 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -40,6 +40,11 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true + - name: Configure Poetry JFrog source + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From ee9dda0bd9bd053ac6c9a021bb1b039eb7510677 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:04:46 +0000 Subject: [PATCH 07/10] Run poetry lock --no-update after adding JFrog source Adding a Poetry source modifies pyproject.toml which invalidates the lock file hash. Running poetry lock --no-update regenerates the lock metadata without changing dependency versions. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 2 ++ .github/workflows/integration.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index b532615..0d95a54 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -41,6 +41,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 @@ -83,6 +84,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9201c9f..4d6cd0e 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -45,6 +45,7 @@ jobs: poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock --no-update - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From f1ea86907b729c9915eca0a4f498e0bb999079fb Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:06:43 +0000 Subject: [PATCH 08/10] Use legacy pip installer for Poetry to respect PIP_INDEX_URL Poetry's modern installer resolves directly from pypi.org, ignoring PIP_INDEX_URL. Setting installer.modern-installation=false makes Poetry use pip for installation, which respects PIP_INDEX_URL and routes through JFrog. This avoids modifying pyproject.toml in CI. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 14 ++------------ .github/workflows/integration.yml | 7 +------ 2 files changed, 3 insertions(+), 18 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 0d95a54..8ebc6f1 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -36,12 +36,7 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - - name: Configure Poetry JFrog source - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry config installer.modern-installation false - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 @@ -79,12 +74,7 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - - name: Configure Poetry JFrog source - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry config installer.modern-installation false - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 4d6cd0e..264a750 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -40,12 +40,7 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - - name: Configure Poetry JFrog source - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock --no-update + poetry config installer.modern-installation false - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From c8bb83f3ffaf7cecca096f743d3a1201fd63ae31 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:18:47 +0000 Subject: [PATCH 09/10] Use poetry source add + poetry lock to resolve via JFrog Match databricks-sql-python2 pattern: add JFrog as primary Poetry source and run full poetry lock to re-resolve through JFrog proxy. This replaces the installer.modern-installation approach which is not supported in Poetry 2.x. Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/workflows/code-quality-checks.yml | 14 ++++++++++++-- .github/workflows/integration.yml | 7 ++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index 8ebc6f1..b9b74cb 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -36,7 +36,12 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - poetry config installer.modern-installation false + - name: Configure Poetry JFrog source + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 @@ -74,7 +79,12 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - poetry config installer.modern-installation false + - name: Configure Poetry JFrog source + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 264a750..9f3b9e8 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -40,7 +40,12 @@ jobs: poetry config virtualenvs.create true poetry config virtualenvs.in-project true poetry config installer.parallel true - poetry config installer.modern-installation false + - name: Configure Poetry JFrog source + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 From e7dc28bc006779189b8fe995658e1837e22323a6 Mon Sep 17 00:00:00 2001 From: Vikrant Puppala Date: Mon, 13 Apr 2026 10:25:07 +0000 Subject: [PATCH 10/10] Extract shared Poetry+JFrog setup into composite action Create .github/actions/setup-poetry that encapsulates JFrog setup, Python setup, Poetry install, JFrog source config, caching, and dependency installation. This removes duplication across all three workflows (check-linting, check-types, integration). Co-authored-by: Isaac Signed-off-by: Vikrant Puppala --- .github/actions/setup-poetry/action.yml | 55 +++++++++++++++++++++ .github/workflows/code-quality-checks.yml | 60 ++--------------------- .github/workflows/integration.yml | 28 ++--------- 3 files changed, 62 insertions(+), 81 deletions(-) create mode 100644 .github/actions/setup-poetry/action.yml diff --git a/.github/actions/setup-poetry/action.yml b/.github/actions/setup-poetry/action.yml new file mode 100644 index 0000000..e20b920 --- /dev/null +++ b/.github/actions/setup-poetry/action.yml @@ -0,0 +1,55 @@ +name: Setup Poetry with JFrog +description: Install Poetry, configure JFrog as primary PyPI source, and install project dependencies + +inputs: + python-version: + description: Python version to set up + required: true + install-args: + description: Extra arguments for poetry install (e.g. --all-extras) + required: false + default: "" + +runs: + using: composite + steps: + - name: Setup JFrog + uses: ./.github/actions/setup-jfrog + + - name: Set up python ${{ inputs.python-version }} + id: setup-python + uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + with: + python-version: ${{ inputs.python-version }} + + - name: Install Poetry + shell: bash + run: | + pip install poetry==2.2.1 + poetry config virtualenvs.create true + poetry config virtualenvs.in-project true + poetry config installer.parallel true + + - name: Configure Poetry JFrog source + shell: bash + run: | + poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" + poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple + poetry lock + + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + + - name: Install dependencies + if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' + shell: bash + run: poetry install --no-interaction --no-root + + - name: Install library + shell: bash + run: poetry install --no-interaction ${{ inputs.install-args }} diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index b9b74cb..8d5d906 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -23,36 +23,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry JFrog source - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install library - run: poetry install --no-interaction - name: Black run: poetry run black --check src @@ -66,36 +40,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python ${{ matrix.python-version }} - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: python-version: ${{ matrix.python-version }} - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry JFrog source - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true' - run: poetry install --no-interaction --no-root - - name: Install library - run: poetry install --no-interaction - name: Mypy run: | mkdir .mypy_cache diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml index 9f3b9e8..797f786 100644 --- a/.github/workflows/integration.yml +++ b/.github/workflows/integration.yml @@ -27,32 +27,10 @@ jobs: steps: - name: Check out repository uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - - name: Setup JFrog - uses: ./.github/actions/setup-jfrog - - name: Set up python - id: setup-python - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5 + - name: Setup Poetry + uses: ./.github/actions/setup-poetry with: python-version: "3.10" - - name: Install Poetry - run: | - pip install poetry==2.2.1 - poetry config virtualenvs.create true - poetry config virtualenvs.in-project true - poetry config installer.parallel true - - name: Configure Poetry JFrog source - run: | - poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}" - poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple - poetry lock - - name: Load cached venv - id: cached-poetry-dependencies - uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4 - with: - path: .venv - key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} - - name: Install dependencies - run: poetry install --no-interaction --all-extras + install-args: "--all-extras" - name: Run SQL Alchemy tests run: poetry run python -m pytest tests/test_local