Fix Persistent Integrations for RPM packages

omnibus/package-scripts/agent-deb/postinst

@@ -187,7 +187,7 @@ else
 fi
 
 if [ -f "$INSTALL_DIR/embedded/bin/python" ]; then
-    ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/postinst.py" "${INSTALL_DIR}" || true
+    ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/post.py" "${INSTALL_DIR}" || true
 fi
 
 exit 0

omnibus/package-scripts/agent-deb/prerm

@@ -156,11 +156,11 @@ remove_remote_config_db()
 remove_persist_integration_files()
 {
     # Remove any file related to reinstalling non-core integrations (see python-scripts/packages.py for the names)
-    if [ -f "$INSTALL_DIR/.prerm_python_installed_packages.txt" ]; then
-        rm "$INSTALL_DIR/.prerm_python_installed_packages.txt" || true
+    if [ -f "$INSTALL_DIR/.pre_python_installed_packages.txt" ]; then
+        rm "$INSTALL_DIR/.pre_python_installed_packages.txt" || true
     fi
-    if [ -f "$INSTALL_DIR/.postinst_python_installed_packages.txt" ]; then
-        rm "$INSTALL_DIR/.postinst_python_installed_packages.txt" || true
+    if [ -f "$INSTALL_DIR/.post_python_installed_packages.txt" ]; then
+        rm "$INSTALL_DIR/.post_python_installed_packages.txt" || true
     fi
     if [ -f "$INSTALL_DIR/.diff_python_installed_packages.txt" ]; then
         rm "$INSTALL_DIR/.diff_python_installed_packages.txt" || true
@@ -178,7 +178,7 @@ case "$1" in #this can't be merged with the later case block because running 're
     upgrade)
         # We're upgrading.
         if [ -f "$INSTALL_DIR/embedded/bin/python" ]; then
-            ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/prerm.py" "${INSTALL_DIR}" || true
+            ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/pre.py" "${INSTALL_DIR}" || true
         fi
     ;;
     *)

omnibus/package-scripts/agent-rpm/postinst

@@ -41,7 +41,7 @@ chown -R root:root ${INSTALL_DIR}/embedded/share/system-probe/ebpf
 chown -R root:root ${INSTALL_DIR}/embedded/share/system-probe/java
 
 if [ -f "$INSTALL_DIR/embedded/bin/python" ]; then
-    ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/postinst.py" "${INSTALL_DIR}" || true
+    ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/post.py" "${INSTALL_DIR}" || true
 fi
 
 exit 0

omnibus/package-scripts/agent-rpm/preinst

@@ -53,6 +53,24 @@ if ! getent passwd dd-agent >/dev/null; then
     fi
 fi
 
+
+
+if [ -f "$INSTALL_DIR/embedded/bin/python" ]; then
+    if [ -f "${INSTALL_DIR}/python-scripts/pre.py" ]; then
+        PRE_PYTHON_FILE="${INSTALL_DIR}/python-scripts/pre.py"
+    else
+        # Search for previous version of the python file
+        if [ -f "${INSTALL_DIR}/python-scripts/prerm.py" ]; then
+            PRE_PYTHON_FILE="${INSTALL_DIR}/python-scripts/prerm.py"
+        fi
+    fi
+    if [ -n "$PRE_PYTHON_FILE" ]; then
+        ${INSTALL_DIR}/embedded/bin/python "${PRE_PYTHON_FILE}" "${INSTALL_DIR}" || true
+    else
+        echo "[ WARNING ]\tPRE_PYTHON_FILE is not set"
+    fi
+fi
+
 # Starting with 6.10, integrations are also uninstalled on package removal
 
 # Since 6.18.0, a file containing all integrations files which have been installed by

omnibus/package-scripts/agent-rpm/prerm

@@ -140,11 +140,11 @@ remove_remote_config_db()
 remove_persist_integration_files()
 {
     # Remove any file related to reinstalling non-core integrations (see python-scripts/packages.py for the names)
-    if [ -f "$INSTALL_DIR/.prerm_python_installed_packages.txt" ]; then
-        rm "$INSTALL_DIR/.prerm_python_installed_packages.txt" || true
+    if [ -f "$INSTALL_DIR/.pre_python_installed_packages.txt" ]; then
+        rm "$INSTALL_DIR/.pre_python_installed_packages.txt" || true
     fi
-    if [ -f "$INSTALL_DIR/.postinst_python_installed_packages.txt" ]; then
-        rm "$INSTALL_DIR/.postinst_python_installed_packages.txt" || true
+    if [ -f "$INSTALL_DIR/.post_python_installed_packages.txt" ]; then
+        rm "$INSTALL_DIR/.post_python_installed_packages.txt" || true
     fi
     if [ -f "$INSTALL_DIR/.diff_python_installed_packages.txt" ]; then
         rm "$INSTALL_DIR/.diff_python_installed_packages.txt" || true
@@ -158,16 +158,6 @@ remove_fips_module()
     rm -rf "${INSTALL_DIR}/embedded/ssl/fipsmodule.cnf" || true
 }
 
-case "$*" in
-    1)
-        # We're upgrading.
-        if [ -f "$INSTALL_DIR/embedded/bin/python" ]; then
-            ${INSTALL_DIR}/embedded/bin/python "${INSTALL_DIR}/python-scripts/prerm.py" "${INSTALL_DIR}" || true
-        fi
-    ;;
-    *)
-    ;;
-esac
 stop_agent
 deregister_agent
 remove_sysprobe_core_files

omnibus/python-scripts/packages.py

@@ -2,10 +2,12 @@
 import pwd
 import grp
 import importlib.metadata
-import pkg_resources
-from packaging import version
+import packaging
 import subprocess
 
+import packaging.requirements
+import packaging.version
+
 DO_NOT_REMOVE_WARNING_HEADER = "# DO NOT REMOVE/MODIFY - used internally by installation process\n"
 
 def run_command(command):
@@ -23,25 +25,28 @@ def run_command(command):
 
 def extract_version(specifier):
     """
-    Extract version from the specifier string.
+    Extract version from the specifier string using packaging.
     """
     try:
-        # Get the first version specifier from the specifier string
-        return str(next(iter(pkg_resources.Requirement.parse(f'{specifier}').specifier)))
-    except Exception:
+        # Parse the specifier and get the first version from the specifier set
+        requirement = packaging.requirements.Requirement(specifier)
+        version = next(iter(requirement.specifier), None)
+        return str(version) if version else None
+    except Exception as e:
+        print(f"Error parsing specifier: {e}")
         return None
 
-def prerm_python_installed_packages_file(directory):
+def pre_python_installed_packages_file(directory):
     """
-    Create prerm installed packages file path.
+    Create pre installed packages file path.
     """
-    return os.path.join(directory, '.prerm_python_installed_packages.txt')
+    return os.path.join(directory, '.pre_python_installed_packages.txt')
 
-def postinst_python_installed_packages_file(directory):
+def post_python_installed_packages_file(directory):
     """
-    Create postinst installed packages file path.
+    Create post installed packages file path.
     """
-    return os.path.join(directory, '.postinst_python_installed_packages.txt')
+    return os.path.join(directory, '.post_python_installed_packages.txt')
 
 def diff_python_installed_packages_file(directory):
     """
@@ -85,7 +90,7 @@ def create_diff_installed_packages_file(directory, old_file, new_file):
                 old_version_str = extract_version(str(old_req_value.specifier))
                 new_version_str = extract_version(str(new_req_value.specifier))
                 if old_version_str and new_version_str:
-                    if version.parse(new_version_str) > version.parse(old_version_str):
+                    if packaging.version.parse(new_version_str) > packaging.version.parse(old_version_str):
                         f.write(f"{new_req_value}\n")
             else:
                 # Package is new in the new file; include it
@@ -154,7 +159,11 @@ def load_requirements(filename):
             else:
                 # Add valid requirement to the list
                 valid_requirements.append(req_stripped)
-    return {req.name: (req_stripped, req) for req_stripped, req in zip(valid_requirements, pkg_resources.parse_requirements(valid_requirements))}
+    # Parse valid requirements using packaging
+    return {
+        req.name: (req_stripped, req)
+        for req_stripped, req in zip(valid_requirements, (packaging.requirements.Requirement(r) for r in valid_requirements))
+    }
 
 def cleanup_files(*files):
     """

omnibus/python-scripts/postinst.py → omnibus/python-scripts/post.py

@@ -3,7 +3,7 @@
 
 Usage:
 - The script should be run with a single argument specifying the installation directory.
-- Example: `python postinst.py /path/to/install/dir`
+- Example: `python post.py /path/to/install/dir`
 """
 
 import os
@@ -13,11 +13,11 @@
 def main():
     try:
         if len(sys.argv) != 2:
-            print("Usage: postinst.py <INSTALL_DIR>")
+            print("Usage: post.py <INSTALL_DIR>")
         install_directory = sys.argv[1]
         if os.path.exists(install_directory):
-            postinst_python_installed_packages_file = packages.postinst_python_installed_packages_file(install_directory)
-            packages.create_python_installed_packages_file(postinst_python_installed_packages_file)
+            post_python_installed_packages_file = packages.post_python_installed_packages_file(install_directory)
+            packages.create_python_installed_packages_file(post_python_installed_packages_file)
             flag_path = f"{install_directory}/.install_python_third_party_deps"
             if os.path.exists(flag_path):
                 print(f"File '{flag_path}' found")

omnibus/python-scripts/pre.py

@@ -0,0 +1,33 @@
+"""
+This module handles the cleanup of Datadog integrations and Python dependencies during package removal.
+
+Usage:
+- The script should be run with a single argument specifying the installation directory.
+- Example: `python pre.py /path/to/install/dir`
+"""
+
+import os
+import sys
+import packages
+
+def main():
+    try:
+        if len(sys.argv) != 2:
+            print("Usage: pre.py <INSTALL_DIR>")
+        install_directory = sys.argv[1]
+        if os.path.exists(install_directory):
+            post_python_installed_packages_file = packages.post_python_installed_packages_file(install_directory)
+            if os.path.exists(post_python_installed_packages_file):
+                pre_python_installed_packages_file = packages.pre_python_installed_packages_file(install_directory)
+                packages.create_python_installed_packages_file(pre_python_installed_packages_file)
+                packages.create_diff_installed_packages_file(install_directory, post_python_installed_packages_file, pre_python_installed_packages_file)
+                packages.cleanup_files(post_python_installed_packages_file, pre_python_installed_packages_file)
+            else:
+                print(f"File {post_python_installed_packages_file} does not exist.")
+        else:
+            print(f"Directory {install_directory} does not exist.")
+    except Exception as e:
+        print(f"Error: {e}")
+
+if __name__ == '__main__':
+    main()

releasenotes/notes/fix_rpm_persisting_integrations-81c2b6333b12edb6.yaml

@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    Fixed a bug that occurs when reinstalling marketplace/extra integrations for a RPM package after an Agent upgrade.