[OSS ONLY] Cherrypicking caching to 4X (#3432)

his commit implements a more efficient method of caching compilations. Cache now restored will now be guaranteed to compatible with the version and build configuration of compilations. We will now save 5-8 mins in version upgrade and dump restore tests. Key changes: Implemented new format for cache keys which includes github hashcode of corresponding engine branch and build mode to better identify the cache stored in a key. New format "ccache-engineHashcode-buildMode-extensionHashcode". Cache restoration handled by build-modified-postgres.yml and saving by save-ccache.yml composite actions. Removed timeStamp based caching. Task: BABEL-5564 Signed-off-by: SiddharthBITS <[email protected]>
babelfish-for-postgresql · Jan 30, 2025 · 31c5413 · 31c5413
1 parent 11c3287
commit 31c5413
Show file tree

Hide file tree

Showing 12 changed files with 131 additions and 24 deletions.
diff --git a/.github/composite-actions/build-modified-postgres/action.yml b/.github/composite-actions/build-modified-postgres/action.yml
@@ -24,7 +24,7 @@ inputs:
 runs:
   using: "composite"
   steps:
-    - name: Checkout, Build, and Install the Modified PostgreSQL Instance and Run Tests
+    - name: Checkout Modified PostgreSQL for Babelfish
       run: |
         cd ..
         rm -rf postgresql_modified_for_babelfish
@@ -44,10 +44,47 @@ runs:
           fi
           REPOSITORY_OWNER=$GITHUB_REPOSITORY_OWNER
         fi
-
         $GITHUB_WORKSPACE/.github/scripts/clone_engine_repo "$REPOSITORY_OWNER" "$ENGINE_BRANCH"
         cd postgresql_modified_for_babelfish
-        git rev-parse HEAD
+        rm -rf ~/.ccache
+        if [[ ${{inputs.tap_tests}} == "yes" ]]; then
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-tapTest" >> $GITHUB_ENV
+        elif [[ ${{inputs.code_coverage}} == "yes" ]]; then
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-coverage" >> $GITHUB_ENV
+        elif [[ ${{inputs.release_mode}} == "yes" ]]; then
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-release" >> $GITHUB_ENV
+        else
+          echo "CRESTORE_KEY=$(git rev-parse --short HEAD)-default" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - uses: actions/cache/restore@v4
+      id: restore-ccache
+      if: ${{ github.event_name == 'pull_request' }} || ${{ inputs.code_coverage == 'yes' }}
+      with:
+        path: 
+          ~/.ccache
+        key: 
+          random-random
+        restore-keys: 
+          ccache-${{ env.CRESTORE_KEY }}
+
+    - name: Save cache if cache hit fails in pull requests
+      if: ${{ github.event_name == 'pull_request' }}
+      run: |
+        if [[ '${{ steps.restore-ccache.outputs.cache-matched-key }}' == '' ]]; then
+          echo "SAVE_CCACHE=1" >> $GITHUB_ENV
+        elif [[ ${{ 'steps.restore-ccache.outputs.cache-matched-key' }} != '' ]]; then
+          echo "SAVE_CCACHE=" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - name: Build and Install the Modified PostgreSQL Instance and Run Tests
+      run: |
+        echo "${{env.SAVE_CCACHE}}"
+        echo "${{ steps.restore-ccache.outputs.cache-matched-key }}"
+        cd ..
+        cd postgresql_modified_for_babelfish
         if [[ ${{inputs.tap_tests}} == "yes" ]]; then
           ./configure CC='ccache gcc' --prefix=$HOME/${{ inputs.install_dir }}/ --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu --enable-tap-tests --with-gssapi
         elif [[ ${{inputs.code_coverage}} == "yes" ]]; then

diff --git a/.github/composite-actions/dump-restore-util/action.yml b/.github/composite-actions/dump-restore-util/action.yml
@@ -327,6 +327,10 @@ runs:
         sqlcmd -S localhost -U jdbc_user -P 12345678 -Q "SELECT @@version GO"
       shell: bash
 
+    - name: Save cache
+      if: always()
+      uses: ./.github/composite-actions/save-ccache
+
     - name: Run Verify Tests
       if: always() && steps.run-pg_dump-restore.outcome == 'success' && inputs.is_final_ver == 'true'
       uses: ./.github/composite-actions/run-verify-tests

diff --git a/.github/composite-actions/install-dependencies/action.yml b/.github/composite-actions/install-dependencies/action.yml
@@ -15,14 +15,4 @@ runs:
         sudo /usr/sbin/update-ccache-symlinks
         echo 'export PATH="/usr/lib/ccache:$PATH"' | tee -a ~/.bashrc
         source ~/.bashrc && echo $PATH
-        echo "NOW=$(date +'%Y-%m-%dT%H:%M:%S')" >> $GITHUB_ENV
       shell: bash
-
-    - name: Restore ccache
-      id: cache-compiler
-      uses: actions/cache@v3
-      with:
-        path: ~/.ccache
-        key: ccache-${{ runner.os }}-${{ env.NOW }}
-        restore-keys: |
-          ccache-${{ runner.os }}
diff --git a/.github/composite-actions/minor-version-upgrade-util/action.yml b/.github/composite-actions/minor-version-upgrade-util/action.yml
@@ -48,6 +48,10 @@ runs:
         install_dir: ${{ inputs.install_dir }}
         extension_branch: ${{ inputs.extension_branch }}
 
+    - name: Save cache
+      if: always() && steps.build-extensions-newer == 'success'
+      uses: ./.github/composite-actions/save-ccache
+
     # Not created and used composite action update-extensions here since, in the previous step it has 
     # checked out a branch/tag which may not have the updated update-extension composite action
     - name: Update extensions

diff --git a/.github/composite-actions/save-ccache/action.yml b/.github/composite-actions/save-ccache/action.yml
@@ -0,0 +1,39 @@
+name: 'Save ccache on push'
+
+runs:
+  using: "composite"
+  steps:
+
+    - name: Setup new cache key
+      if: always()
+      run: |
+        echo "CCACHE_KEY=${{env.CRESTORE_KEY}}-$(git rev-parse --short HEAD)" >> $GITHUB_ENV
+        echo "Event : ${{ github.event_name }}"
+        echo "Save ccache : ${{ env.SAVE_CCACHE }}"
+        if [[ ${{ github.event_name != 'pull_request' || env.SAVE_CCACHE == 1 }} == true ]]; then
+          echo "Cache to be Saved"
+          echo "SAVE_CACHE_HERE=1" >> $GITHUB_ENV
+        else
+          echo "Cache shouldn't be Saved"
+          echo "SAVE_CACHE_HERE=0" >> $GITHUB_ENV
+        fi
+      shell: bash
+
+    - name: Save ccache
+      uses: actions/cache/save@v4
+      if: env.SAVE_CACHE_HERE == 1
+      with:
+        path: 
+          ~/.ccache
+        key: 
+          ccache-${{ env.CCACHE_KEY }}
+
+    - name: Clean ccache directory and unset env variables
+      if: always()
+      shell: bash
+      run: |
+        rm -rf ~/.ccache
+        echo "CCACHE_KEY=" >> $GITHUB_ENV
+        echo "SAVE_CCACHE=" >> $GITHUB_ENV
+        echo "SAVE_CACHE_HERE=" >> $GITHUB_ENV
+        echo "CRESTORE_KEY=" >> $GITHUB_ENV
diff --git a/.github/composite-actions/setup-base-version/action.yml b/.github/composite-actions/setup-base-version/action.yml
@@ -222,6 +222,10 @@ runs:
 
     - uses: actions/checkout@v2
 
+    - name: Save cache
+      if: always() && steps.jdbc-upgrade-tests.outcome == 'success'
+      uses: ./.github/composite-actions/save-ccache
+
     - name: Install Python
       id: install-python
       if: ${{ matrix.upgrade-path.path[0] == 'source_latest' && steps.jdbc-upgrade-tests.outcome == 'success' }}

diff --git a/.github/composite-actions/setup-new-version/action.yml b/.github/composite-actions/setup-new-version/action.yml
@@ -64,6 +64,10 @@ runs:
       with:
         install_dir: ${{ inputs.pg_new_dir }}
 
+    - name: Save cache
+      if: always() && steps.build-postgis-extension == 'success'
+      uses: ./.github/composite-actions/save-ccache
+
     - name: Setup new data directory
       id: setup-new-datadir
       if: always() && steps.build-postgis-extension.outcome == 'success'

diff --git a/.github/workflows/dotnet-tests.yml b/.github/workflows/dotnet-tests.yml
@@ -40,6 +40,10 @@ jobs:
         id: install-extensions
         if: always() && steps.build-postgis-extension.outcome == 'success'
         uses: ./.github/composite-actions/install-extensions
+
+      - name: Save cache
+        if: always() && steps.install-extensions.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
 
       - name: Run Dotnet Tests
         id: run-dotnet-tests

diff --git a/.github/workflows/jdbc-tests-single-db-mode.yml b/.github/workflows/jdbc-tests-single-db-mode.yml
@@ -102,6 +102,10 @@ jobs:
             ~/psql/data/logfile
             ~/psql/data_5433/logfile
 
+      - name: Save cache
+        if: always() && steps.replication.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       # The test summary files contain paths with ':' characters, which is not allowed with the upload-artifact actions
       - name: Rename Test Summary Files
         id: test-file-rename

diff --git a/.github/workflows/major-version-upgrade.yml b/.github/workflows/major-version-upgrade.yml
@@ -20,18 +20,11 @@ jobs:
 
       - name: Build Modified Postgres using ${{env.ENGINE_BRANCH_FROM}}
         id: build-modified-postgres-old
+        uses: ./.github/composite-actions/build-modified-postgres
         if: always() && steps.install-dependencies.outcome == 'success'
-        run: |
-          cd ..
-          git clone --branch ${{env.ENGINE_BRANCH_FROM}} https://github.com/babelfish-for-postgresql/postgresql_modified_for_babelfish.git
-          cd postgresql_modified_for_babelfish
-          ./configure --prefix=$HOME/${{env.OLD_INSTALL_DIR}} --with-python PYTHON=/usr/bin/python3.8 --enable-cassert CFLAGS="-ggdb" --with-libxml --with-uuid=ossp --with-icu
-          make clean
-          make -j 4 2>error.txt
-          make install
-          make check
-          cd contrib && make && sudo make install
-        shell: bash
+        with:
+          engine_branch: ${{ env.ENGINE_BRANCH_FROM }}
+          install_dir: ${{ env.OLD_INSTALL_DIR }}
 
       - name: Compile ANTLR
         id: compile-antlr
@@ -130,6 +123,10 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: Save cache
+        if: always() && steps.build-extensions-old.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       - name: Build Modified Postgres using latest version
         id: build-modified-postgres-new
         if: always() && steps.install-extensions-old.outcome == 'success'
@@ -172,6 +169,10 @@ jobs:
         uses: ./.github/composite-actions/build-postgis-extension
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}
+
+      - name: Save cache
+        if: always() && steps.build-postgis-extension.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
 
       - name: Setup new data directory
         id: setup-new-datadir

diff --git a/.github/workflows/minor-version-upgrade.yml b/.github/workflows/minor-version-upgrade.yml
@@ -113,6 +113,10 @@ jobs:
         if: always() && steps.build-tds_fdw-extension.outcome == 'success'
         uses: ./.github/composite-actions/install-extensions
 
+      - name: Save cache
+        if: always() && steps.build-extensions-older.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       - name: Build and run tests for Postgres engine using latest engine
         id: build-modified-postgres-newer
         if: always() && steps.install-extensions-older.outcome == 'success'

diff --git a/.github/workflows/tap-tests.yml b/.github/workflows/tap-tests.yml
@@ -90,6 +90,8 @@ jobs:
           sudo make USE_PGXS=1 PG_CONFIG=~/psql_source/bin/pg_config install
         shell: bash
 
+
+
       - name: Build Extensions using ${{env.EXTENSION_BRANCH_OLD}}
         id: build-extensions-old
         if: always() && steps.build-postgis-extension-old.outcome == 'success'
@@ -100,13 +102,19 @@ jobs:
 
       - uses: actions/checkout@v2
 
+      - name: Save cache
+        if: always() && steps.build-extensions-old.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       - name: Build Modified Postgres using latest version
         id: build-modified-postgres-new
         if: always() && steps.build-extensions-old.outcome == 'success'
         uses: ./.github/composite-actions/build-modified-postgres
         with:
           tap_tests: 'yes'
           install_dir: ${{env.NEW_INSTALL_DIR}}
+
+
 
       - name: Compile new ANTLR
         id: compile-new-antlr
@@ -129,6 +137,10 @@ jobs:
         with:
           install_dir: ${{env.NEW_INSTALL_DIR}}
 
+      - name: Save cache
+        if: always() && steps.build-extensions-old.outcome == 'success'
+        uses: ./.github/composite-actions/save-ccache
+
       - name: Run TAP Tests
         id: tap
         if: always() && steps.build-postgis-extension.outcome == 'success'