move python packages to requirements.txt

Platform specific scripts in contrib/CI are expected to be run as root, but python packages should not be automatically installed into the package manager's directories. They were moved out of the scripts to requirements.txt so that they can be installed without root.
mar-file-system · Feb 20, 2025 · 660cdc9 · 660cdc9
1 parent 6290d24
commit 660cdc9
Show file tree

Hide file tree

Showing 10 changed files with 41 additions and 15 deletions.
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
@@ -69,6 +69,7 @@ env:
   CXX: clang++
   DEP_INSTALL_PREFIX: ~/.local
   COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On
+  PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
   ShellCheck:
@@ -95,10 +96,11 @@ jobs:
     - name: Ubuntu Prerequisites
       run:  sudo contrib/CI/ubuntu.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Install Extra Python Packages
-      run:  |
-            sudo apt -y install python3-pip
-            sudo pip3 install pylint
+      run:  python3 -m pip install pylint
 
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }}
@@ -126,6 +128,9 @@ jobs:
     - name: Install wget
       run:  yum -y install wget
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Uninstall CMake from package manager
       run:  yum -y autoremove cmake3
 
@@ -196,6 +201,9 @@ jobs:
     - name: Ubuntu Prerequisites
       run:  sudo contrib/CI/ubuntu.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }} -DTEST_WORKING_DIRECTORY=/tmp
 
@@ -225,6 +233,9 @@ jobs:
     - name: Install Valgrind
       run:  apt -y install valgrind
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }}
 
@@ -250,6 +261,9 @@ jobs:
     - name: Ubuntu Prerequisites
       run:  sudo contrib/CI/ubuntu.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }} -DDEP_USE_JEMALLOC=Off -DTEST_WORKING_DIRECTORY="$(pwd)"
 

diff --git a/.github/workflows/codecov.yml b/.github/workflows/codecov.yml
@@ -69,6 +69,7 @@ env:
   CXX: g++
   DEP_INSTALL_PREFIX: ~/.local
   COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic -std=c++11" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On -DGCOV=On -DCMAKE_BUILD_TYPE=Debug
+  PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
   CodeCoverage:
@@ -84,6 +85,9 @@ jobs:
     - name: Ubuntu Prerequisites
       run:  sudo contrib/CI/ubuntu.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Coverage Prerequisites
       run:  |
             sudo apt install gcovr

diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml
@@ -69,6 +69,7 @@ env:
   CXX: clang++
   DEP_INSTALL_PREFIX: ~/.local
   COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On
+  PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
   SecurityCheck:
@@ -92,6 +93,9 @@ jobs:
     - name: Ubuntu Prerequisites
       run:  sudo contrib/CI/ubuntu.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }}
 

diff --git a/.github/workflows/rpm.yml b/.github/workflows/rpm.yml
@@ -69,6 +69,7 @@ env:
   CXX: clang++
   DEP_INSTALL_PREFIX: ~/.local
   COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2
+  PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
   RPMs:
@@ -89,6 +90,9 @@ jobs:
     - name: Rocky Linux Prerequisites
       run:  contrib/CI/rockylinux.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  |
             mkdir build

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -69,6 +69,7 @@ env:
   CXX: clang++
   DEP_INSTALL_PREFIX: ~/.local
   COMMON_CONFIG: -DCMAKE_C_FLAGS="-Wall -Wextra -Werror -pedantic" -DCMAKE_CXX_FLAGS="-Wall -Wextra -Werror -pedantic" -DDEP_INSTALL_PREFIX="${DEP_INSTALL_PREFIX}" -DDEP_BUILD_THREADS=2 -DENABLE_SUDO_TESTS=On
+  PIP_BREAK_SYSTEM_PACKAGES: 1
 
 jobs:
   Linux:
@@ -99,6 +100,9 @@ jobs:
       if:   startsWith(matrix.img, 'ubuntu:')
       run:  contrib/CI/ubuntu.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }}
 
@@ -126,6 +130,9 @@ jobs:
     - name: macOS Prerequisites
       run:  contrib/CI/macos.sh
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run:  cmake -B ${{ github.workspace }}/build ${{ env.COMMON_CONFIG }} -DTEST_WORKING_DIRECTORY="$(pwd)"
 
@@ -157,6 +164,9 @@ jobs:
     - name: Set Up Python3
       run:  ln -sf /usr/bin/python3.9 /usr/bin/python3
 
+    - name: Install Python Packages
+      run:  python3 -m pip install -r contrib/CI/requirements.txt
+
     - name: Configure CMake
       run: >-
            export PATH=/usr/bin:$(cygpath ${SYSTEMROOT})/system32 &&

diff --git a/contrib/CI/centos8.sh b/contrib/CI/centos8.sh
@@ -78,6 +78,3 @@ yum -y install attr autoconf clang cmake3 diffutils fuse git make patch pkgconfi
 
 # SQLAlchemy needs Python.h
 yum -y install python3-devel
-
-# install pip packages
-python3 -m pip install pugsql sqlalchemy
diff --git a/contrib/CI/macos.sh b/contrib/CI/macos.sh
@@ -78,6 +78,3 @@ brew install autoconf coreutils cmake pcre2 python
 # get osxfuse from homebrew/cask
 brew tap homebrew/cask
 brew install --cask osxfuse || true # brew 4.0.0 switches to macfuse, but pkg-config can't find it
-
-# install pip packages
-PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install pugsql sqlalchemy
diff --git a/contrib/CI/requirements.txt b/contrib/CI/requirements.txt
@@ -0,0 +1,2 @@
+pugsql
+sqlalchemy
diff --git a/contrib/CI/rockylinux.sh b/contrib/CI/rockylinux.sh
@@ -75,6 +75,3 @@ dnf -y install attr autoconf clang cmake3 findutils fuse git make patch pkgconfi
 
 # SQLAlchemy needs Python.h
 yum -y install python3-devel
-
-# install pip packages
-python3 -m pip install pugsql sqlalchemy
diff --git a/contrib/CI/ubuntu.sh b/contrib/CI/ubuntu.sh
@@ -88,6 +88,3 @@ apt -y install \
     python3 \
     python3-pip \
     sudo
-
-# install pip packages
-PIP_BREAK_SYSTEM_PACKAGES=1 python3 -m pip install pugsql sqlalchemy