Restructure GitHub workflows to use GitHub App tokens

.github/workflows/build-citus-community-nightlies.yml

@@ -4,7 +4,6 @@ env:
   MAIN_BRANCH: "all-citus"
   PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }}
   PACKAGING_SECRET_KEY: ${{ secrets.PACKAGING_SECRET_KEY }}
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
   DOCKERHUB_USER_NAME: ${{ secrets.DOCKERHUB_USER_NAME }}
   DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }}
 on:
@@ -32,9 +31,25 @@ jobs:
           - ubuntu/jammy
 
     steps:
+
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
+          token: ${{ steps.app.outputs.token }}
           fetch-depth: 1
           path: tools
 
@@ -46,7 +61,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Clone build branch
-        run: git clone -b "${MAIN_BRANCH}" --depth=1  https://github.com/citusdata/packaging.git packaging
+        run: git clone -b "${MAIN_BRANCH}" --depth=1 https://github.com/citusdata/packaging.git packaging
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/citus-package-all-platforms-test.yml

@@ -1,8 +1,6 @@
 name: Citus package all platforms tests
 
 env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
-  GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
   PACKAGING_PASSPHRASE: ${{ secrets.PACKAGING_PASSPHRASE }}
   MICROSOFT_EMAIL: gindibay@microsoft.com
   USER_NAME: Gurkan Indibay
@@ -36,8 +34,26 @@ jobs:
       PLATFORM: ${{ matrix.platform }}
 
     steps:
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: |
+          echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Setup git authentication for GitHub App
+        run: |
+          git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          token: ${{ steps.app.outputs.token }}
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/delete-packagecloud-packages.yml

@@ -20,7 +20,7 @@ jobs:
 
     steps:
       - name: Checkout repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/package-tests.yml

@@ -18,10 +18,27 @@ jobs:
       pg_versions: ${{ steps.generate-postgres.outputs.pg_versions }}
       citus_version: ${{ steps.get-citus-version.outputs.citus_version }}
     steps:
+
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout
         uses: actions/checkout@v3
         with:
+          token: ${{ steps.app.outputs.token }}
           fetch-depth: 2
+
       - name: Package version
         id: get-citus-version
         run: |
@@ -60,8 +77,24 @@ jobs:
       PLATFORM: ${{ matrix.platform }}
 
     steps:
-      - name: Checkout repository
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v2
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
+      - name: Checkout
         uses: actions/checkout@v3
+        with:
+          token: ${{ steps.app.outputs.token }}
 
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/packaging-methods-tests.yml

@@ -1,8 +1,5 @@
 name: Packaging helper methods tests
 
-env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
-
 on:
   push:
     branches:
@@ -13,10 +10,25 @@ on:
 jobs:
   unit_test_execution:
     runs-on: ubuntu-latest
-
     steps:
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          token: ${{ steps.app.outputs.token }}
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/publish-docker-image-tests.yml

@@ -10,10 +10,26 @@ on:
 jobs:
   unit_test_execution:
     runs-on: ubuntu-latest
-
     steps:
+
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          token: ${{steps.app.outputs.token}}
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/pypi-statistics-schedule.yml

@@ -21,7 +21,7 @@ jobs:
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3
-
+        
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources
 

.github/workflows/statistic-schedule.yml

@@ -5,7 +5,6 @@ env:
   DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }}
   DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }}
   DB_NAME: ${{ secrets.STATS_DB_NAME }}
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
 on:
   schedule:
     - cron: "0 16 * * *"
@@ -25,8 +24,24 @@ jobs:
         job_name: [docker_pull_citus, github_clone_citus, homebrew_citus]
 
     steps:
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          token: ${{ steps.app.outputs.token }}
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/statistic-tests.yml

@@ -5,7 +5,6 @@ env:
   DB_PASSWORD: ${{ secrets.STATS_DB_PASSWORD }}
   DB_HOST_AND_PORT: ${{ secrets.STATS_DB_HOST_AND_PORT }}
   DB_NAME: ${{ secrets.STATS_DB_NAME }}
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
   PACKAGE_CLOUD_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_API_TOKEN }}
   PACKAGE_CLOUD_ADMIN_API_TOKEN: ${{ secrets.PACKAGE_CLOUD_ADMIN_API_TOKEN }}
 on:
@@ -19,10 +18,26 @@ on:
 jobs:
   unit_test_execution:
     runs-on: ubuntu-latest
-
+    
     steps:
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
+        with:
+          token: ${{ steps.app.outputs.token }}
 
       - name: Install package dependencies
         run: sudo apt-get update && sudo apt-get install libcurl4-openssl-dev libssl-dev python3-testresources

.github/workflows/tool-tests.yml

@@ -1,7 +1,6 @@
 name: Tool Tests
 
 env:
-  GH_TOKEN: ${{ secrets.GH_TOKEN }}
   MICROSOFT_EMAIL: gindibay@microsoft.com
   USER_NAME: Gurkan Indibay
   MAIN_BRANCH: all-citus
@@ -20,16 +19,32 @@ jobs:
     steps:
       - name: Check out repository
         uses: actions/checkout@v3
+
       - name: Install all scripts
         run: make && sudo make install
 
   unit_test_execution:
     runs-on: ubuntu-latest
-
     steps:
+
+      - name: Create GitHub App token
+        id: app
+        uses: actions/create-github-app-token@v1
+        with:
+          app-id: ${{ vars.GH_APP_ID }}
+          private-key: ${{ secrets.GH_APP_KEY }}
+          owner: ${{ github.repository_owner }}
+
+      - name: Set GH_TOKEN for all steps
+        run: echo "GH_TOKEN=${{ steps.app.outputs.token }}" >> $GITHUB_ENV
+
+      - name: Configure git with x-access-token
+        run: git config --global url."https://x-access-token:${{ steps.app.outputs.token }}@github.com/".insteadOf "https://github.com/"
+
       - name: Checkout repository
         uses: actions/checkout@v3
         with:
+          token: ${{ steps.app.outputs.token }}
           fetch-depth: 0
 
       - name: Set up Python 3.10

packaging_automation/common_tool_methods.py

@@ -679,8 +679,11 @@ def remove_suffix(initial_str: str, suffix: str) -> str:
 
 def initialize_env(exec_path: str, project_name: str, checkout_dir: str):
     remove_cloned_code(f"{exec_path}/{checkout_dir}")
+    gh_token = os.getenv("GH_TOKEN")
+    if not gh_token:
+        raise ValueError("GH_TOKEN environment variable is required")
     if not os.path.exists(checkout_dir):
-        run(f"git clone https://github.com/citusdata/{project_name}.git {checkout_dir}")
+        run(f"git clone https://x-access-token:{gh_token}@github.com/citusdata/{project_name}.git {checkout_dir}")
 
 
 def create_pr(

packaging_automation/tests/test_citus_package.py

@@ -61,6 +61,8 @@
 TEST_GPG_KEY_NAME = "Citus Data <packaging@citusdata.com>"
 TEST_GPG_KEY_PASSPHRASE = os.getenv("PACKAGING_PASSPHRASE")
 GH_TOKEN = os.getenv("GH_TOKEN")
+if not GH_TOKEN:
+    raise ValueError("GH_TOKEN environment variable is required for tests to run")
 PACKAGE_CLOUD_API_TOKEN = os.getenv("PACKAGE_CLOUD_API_TOKEN")
 REPO_CLIENT_SECRET = os.getenv("REPO_CLIENT_SECRET")
 PLATFORM = get_build_platform(
@@ -87,8 +89,7 @@ def setup_module():
     )
     if not os.path.exists(PACKAGING_EXEC_FOLDER):
         run(
-            f"git clone --branch {packaging_branch_name} https://github.com/citusdata/packaging.git"
-            f" {PACKAGING_EXEC_FOLDER}"
+            f"git clone --branch {packaging_branch_name} https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_EXEC_FOLDER}"
         )
 
 

packaging_automation/tests/test_citus_package_utils.py

@@ -46,7 +46,7 @@
 def setup_module():
     if not os.path.exists("packaging_test"):
         run(
-            f"git clone --branch all-citus-unit-tests https://github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}"
+            f"git clone --branch all-citus-unit-tests https://x-access-token:{GH_TOKEN}@github.com/citusdata/packaging.git {PACKAGING_SOURCE_FOLDER}"
         )
 
 

packaging_automation/tests/test_common_tool_methods.py

@@ -46,7 +46,7 @@
     str_array_to_str,
 )
 
-GITHUB_TOKEN = os.getenv("GH_TOKEN")
+GH_TOKEN = os.getenv("GH_TOKEN")
 BASE_PATH = pathlib2.Path(__file__).parents[1]
 TEST_BASE_PATH = pathlib2.Path(__file__).parent.absolute()
 TEST_GPG_KEY_NAME = "Citus Data <packaging@citusdata.com>"
@@ -264,7 +264,7 @@ def test_prepend_line_in_file():
 
 def test_getprs():
     # created at is not seen on Github. Should be checked on API result
-    g = Github(GITHUB_TOKEN)
+    g = Github(GH_TOKEN)
     repository = g.get_repo("citusdata/citus")
     prs = get_prs_for_patch_release(
         repository,
@@ -277,7 +277,7 @@ def test_getprs():
 
 
 def test_getprs_with_backlog_label():
-    g = Github(GITHUB_TOKEN)
+    g = Github(GH_TOKEN)
     repository = g.get_repo("citusdata/citus")
     prs = get_prs_for_patch_release(
         repository,