Move CI to Databricks protected runners with JFrog OIDC

.github/actions/setup-jfrog/action.yml

@@ -0,0 +1,32 @@
+name: Setup JFrog OIDC
+description: Obtain a JFrog access token via GitHub OIDC and configure pip to use JFrog PyPI proxy
+
+runs:
+  using: composite
+  steps:
+    - name: Get JFrog OIDC token
+      shell: bash
+      run: |
+        set -euo pipefail
+        ID_TOKEN=$(curl -sLS \
+          -H "User-Agent: actions/oidc-client" \
+          -H "Authorization: Bearer $ACTIONS_ID_TOKEN_REQUEST_TOKEN" \
+          "${ACTIONS_ID_TOKEN_REQUEST_URL}&audience=jfrog-github" | jq .value | tr -d '"')
+        echo "::add-mask::${ID_TOKEN}"
+        ACCESS_TOKEN=$(curl -sLS -XPOST -H "Content-Type: application/json" \
+          "https://databricks.jfrog.io/access/api/v1/oidc/token" \
+          -d "{\"grant_type\": \"urn:ietf:params:oauth:grant-type:token-exchange\", \"subject_token_type\":\"urn:ietf:params:oauth:token-type:id_token\", \"subject_token\": \"${ID_TOKEN}\", \"provider_name\": \"github-actions\"}" | jq .access_token | tr -d '"')
+        echo "::add-mask::${ACCESS_TOKEN}"
+        if [ -z "$ACCESS_TOKEN" ] || [ "$ACCESS_TOKEN" = "null" ]; then
+          echo "FAIL: Could not extract JFrog access token"
+          exit 1
+        fi
+        echo "JFROG_ACCESS_TOKEN=${ACCESS_TOKEN}" >> "$GITHUB_ENV"
+        echo "JFrog OIDC token obtained successfully"
+
+    - name: Configure pip
+      shell: bash
+      run: |
+        set -euo pipefail
+        echo "PIP_INDEX_URL=https://gha-service-account:${JFROG_ACCESS_TOKEN}@databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple" >> "$GITHUB_ENV"
+        echo "pip configured to use JFrog registry"

.github/actions/setup-poetry/action.yml

@@ -0,0 +1,55 @@
+name: Setup Poetry with JFrog
+description: Install Poetry, configure JFrog as primary PyPI source, and install project dependencies
+
+inputs:
+  python-version:
+    description: Python version to set up
+    required: true
+  install-args:
+    description: Extra arguments for poetry install (e.g. --all-extras)
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Setup JFrog
+      uses: ./.github/actions/setup-jfrog
+
+    - name: Set up python ${{ inputs.python-version }}
+      id: setup-python
+      uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Install Poetry
+      shell: bash
+      run: |
+        pip install poetry==2.2.1
+        poetry config virtualenvs.create true
+        poetry config virtualenvs.in-project true
+        poetry config installer.parallel true
+
+    - name: Configure Poetry JFrog source
+      shell: bash
+      run: |
+        poetry config repositories.jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
+        poetry config http-basic.jfrog gha-service-account "${JFROG_ACCESS_TOKEN}"
+        poetry source add --priority=primary jfrog https://databricks.jfrog.io/artifactory/api/pypi/db-pypi/simple
+        poetry lock
+
+    - name: Load cached venv
+      id: cached-poetry-dependencies
+      uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
+      with:
+        path: .venv
+        key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
+
+    - name: Install dependencies
+      if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
+      shell: bash
+      run: poetry install --no-interaction --no-root
+
+    - name: Install library
+      shell: bash
+      run: poetry install --no-interaction ${{ inputs.install-args }}

.github/workflows/code-quality-checks.yml

@@ -10,112 +10,41 @@ on:
 
 permissions:
     contents: read
+    id-token: write
 
 jobs:
     check-linting:
-        runs-on: ubuntu-latest
+        runs-on:
+            group: databricks-protected-runner-group
+            labels: linux-ubuntu-latest
         strategy:
             matrix:
                 python-version: [3.9, "3.10", "3.11", "3.12"]
         steps:
-            #----------------------------------------------
-            #       check-out repo and set-up python
-            #----------------------------------------------
             - name: Check out repository
-              uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
-            - name: Set up python ${{ matrix.python-version }}
-              id: setup-python
-              uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2
+              uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+            - name: Setup Poetry
+              uses: ./.github/actions/setup-poetry
               with:
                   python-version: ${{ matrix.python-version }}
-            #----------------------------------------------
-            #  -----  install & configure poetry  -----
-            #----------------------------------------------
-            - name: Install Poetry
-              uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
-              with:
-                  version: "2.2.1"
-                  virtualenvs-create: true
-                  virtualenvs-in-project: true
-                  installer-parallel: true
-
-            #----------------------------------------------
-            #       load cached venv if cache exists
-            #----------------------------------------------
-            - name: Load cached venv
-              id: cached-poetry-dependencies
-              uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-              with:
-                  path: .venv
-                  key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
-            #----------------------------------------------
-            # install dependencies if cache does not exist
-            #----------------------------------------------
-            - name: Install dependencies
-              if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-              run: poetry install --no-interaction --no-root
-            #----------------------------------------------
-            # install your root project, if required
-            #----------------------------------------------
-            - name: Install library
-              run: poetry install --no-interaction
-            #----------------------------------------------
-            #              black the code
-            #----------------------------------------------
             - name: Black
               run: poetry run black --check src
 
     check-types:
-        runs-on: ubuntu-latest
+        runs-on:
+            group: databricks-protected-runner-group
+            labels: linux-ubuntu-latest
         strategy:
             matrix:
                 python-version: [3.9, "3.10", "3.11", "3.12"]
         steps:
-            #----------------------------------------------
-            #       check-out repo and set-up python
-            #----------------------------------------------
             - name: Check out repository
-              uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2
-            - name: Set up python ${{ matrix.python-version }}
-              id: setup-python
-              uses: actions/setup-python@e9aba2c848f5ebd159c070c61ea2c4e2b122355e # v2
+              uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+            - name: Setup Poetry
+              uses: ./.github/actions/setup-poetry
               with:
                   python-version: ${{ matrix.python-version }}
-            #----------------------------------------------
-            #  -----  install & configure poetry  -----
-            #----------------------------------------------
-            - name: Install Poetry
-              uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
-              with:
-                  version: "2.2.1"
-                  virtualenvs-create: true
-                  virtualenvs-in-project: true
-                  installer-parallel: true
-
-            #----------------------------------------------
-            #       load cached venv if cache exists
-            #----------------------------------------------
-            - name: Load cached venv
-              id: cached-poetry-dependencies
-              uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-              with:
-                  path: .venv
-                  key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
-            #----------------------------------------------
-            # install dependencies if cache does not exist
-            #----------------------------------------------
-            - name: Install dependencies
-              if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
-              run: poetry install --no-interaction --no-root
-            #----------------------------------------------
-            # install your root project, if required
-            #----------------------------------------------
-            - name: Install library
-              run: poetry install --no-interaction
-            #----------------------------------------------
-            #              mypy the code
-            #----------------------------------------------
             - name: Mypy
               run: |
-                  mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153
-                  poetry run mypy --install-types --non-interactive src
+                  mkdir .mypy_cache
+                  poetry run mypy --install-types --non-interactive src

.github/workflows/dco-check.yml

@@ -1,29 +1,74 @@
 name: DCO Check
 
-on: [pull_request]
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+    branches: [main]
 
 permissions:
-    contents: read
-    pull-requests: write
+  contents: read
 
 jobs:
-    check:
-        runs-on: ubuntu-latest
-        steps:
-            - name: Check for DCO
-              id: dco-check
-              uses: tisonkun/actions-dco@6d1f8a197db1b04df1769707b46b9366b1eca902 # v1.1
-            - name: Comment about DCO status
-              uses: actions/github-script@00f12e3e20659f42342b1c0226afda7f7c042325 # v6
-              if: ${{ failure() }}
-              with:
-                  script: |
-                      github.rest.issues.createComment({
-                        issue_number: context.issue.number,
-                        owner: context.repo.owner,
-                        repo: context.repo.repo,
-                        body: `Thanks for your contribution! To satisfy the DCO policy in our \
-                        [contributing guide](https://github.com/databricks/databricks-sqlalchemy/blob/main/CONTRIBUTING.md) \
-                        every commit message must include a sign-off message. One or more of your commits is missing this message. \
-                        You can reword previous commit messages with an interactive rebase (\`git rebase -i main\`).`
-                      })
+  dco-check:
+    runs-on:
+      group: databricks-protected-runner-group
+      labels: linux-ubuntu-latest
+    name: Check DCO Sign-off
+    steps:
+      - name: Checkout
+        uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+        with:
+          fetch-depth: 0
+
+      - name: Check DCO Sign-off
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha }}
+          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
+        run: |
+          #!/bin/bash
+          set -e
+
+          echo "Checking commits from $BASE_SHA to $HEAD_SHA"
+
+          COMMITS=$(git rev-list --no-merges "$BASE_SHA..$HEAD_SHA")
+
+          if [ -z "$COMMITS" ]; then
+            echo "No commits found in this PR"
+            exit 0
+          fi
+
+          FAILED_COMMITS=()
+
+          for commit in $COMMITS; do
+            echo "Checking commit: $commit"
+            COMMIT_MSG=$(git log --format=%B -n 1 "$commit")
+            if echo "$COMMIT_MSG" | grep -q "^Signed-off-by: "; then
+              echo "  Commit $commit has DCO sign-off"
+            else
+              echo "  Commit $commit is missing DCO sign-off"
+              FAILED_COMMITS+=("$commit")
+            fi
+          done
+
+          if [ ${#FAILED_COMMITS[@]} -ne 0 ]; then
+            echo ""
+            echo "DCO Check Failed!"
+            echo "The following commits are missing the required 'Signed-off-by' line:"
+            for commit in "${FAILED_COMMITS[@]}"; do
+              echo "  - $commit: $(git log --format=%s -n 1 "$commit")"
+            done
+            echo ""
+            echo "To fix this, you need to sign off your commits. You can:"
+            echo "1. Add sign-off to new commits: git commit -s -m 'Your commit message'"
+            echo "2. Amend existing commits: git commit --amend --signoff"
+            echo "3. For multiple commits, use: git rebase --signoff HEAD~N (where N is the number of commits)"
+            echo ""
+            echo "The sign-off should be in the format:"
+            echo "Signed-off-by: Your Name <your.email@example.com>"
+            echo ""
+            echo "For more details, see CONTRIBUTING.md"
+            exit 1
+          else
+            echo ""
+            echo "All commits have proper DCO sign-off!"
+          fi

.github/workflows/integration.yml

@@ -9,10 +9,13 @@ on:
 
 permissions:
     contents: read
+    id-token: write
 
 jobs:
     run-e2e-tests:
-        runs-on: ubuntu-latest
+        runs-on:
+            group: databricks-protected-runner-group
+            labels: linux-ubuntu-latest
         environment: azure-prod
         env:
             DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }}
@@ -22,43 +25,12 @@ jobs:
             DATABRICKS_SCHEMA: ${{ secrets.SQLALCHEMY_SCHEMA }}
             DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }}
         steps:
-            #----------------------------------------------
-            #       check-out repo and set-up python
-            #----------------------------------------------
             - name: Check out repository
-              uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3
-            - name: Set up python
-              id: setup-python
-              uses: actions/setup-python@7f4fc3e22c37d6ff65e88745f38bd3157c663f7c # v4
+              uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
+            - name: Setup Poetry
+              uses: ./.github/actions/setup-poetry
               with:
                   python-version: "3.10"
-            #----------------------------------------------
-            #  -----  install & configure poetry  -----
-            #----------------------------------------------
-            - name: Install Poetry
-              uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
-              with:
-                  version: "2.2.1"
-                  virtualenvs-create: true
-                  virtualenvs-in-project: true
-                  installer-parallel: true
-
-            #----------------------------------------------
-            #       load cached venv if cache exists
-            #----------------------------------------------
-            - name: Load cached venv
-              id: cached-poetry-dependencies
-              uses: actions/cache@0057852bfaa89a56745cba8c7296529d2fc39830 # v4
-              with:
-                  path: .venv
-                  key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }}
-            #----------------------------------------------
-            # install dependencies if cache does not exist
-            #----------------------------------------------
-            - name: Install dependencies
-              run: poetry install --no-interaction --all-extras
-            #----------------------------------------------
-            #              run test suite
-            #----------------------------------------------
+                  install-args: "--all-extras"
             - name: Run SQL Alchemy tests
               run: poetry run python -m pytest tests/test_local