Support compressed comps.xml file

Signed-off-by: Mattia Verga <[email protected]>

bodhi-server/bodhi-server.spec

@@ -25,7 +25,9 @@ BuildRequires:  python3-sphinx
 BuildRequires:  python3-responses
 BuildRequires:  python3-webtest
 BuildRequires:  python3-librepo
+BuildRequires:  python3dist(libcomps) >= 0.1.20
 BuildRequires:  python3-createrepo_c
+BuildRequires:  python3-zstandard
 BuildRequires:  createrepo_c
 BuildRequires:  skopeo
 BuildRequires:  dnf

bodhi-server/bodhi/server/util.py

@@ -22,10 +22,13 @@
 from datetime import datetime, timedelta
 from importlib import import_module
 from urllib.parse import urlencode
+import bz2
 import errno
 import functools
+import gzip
 import hashlib
 import json
+import lzma
 import os
 import re
 import socket
@@ -47,6 +50,7 @@
 import pkg_resources
 import requests
 import rpm
+import zstandard
 
 from bodhi.server import ffmarkdown, log, buildsys, Session
 from bodhi.server.config import config
@@ -320,7 +324,21 @@ def sanity_check_repodata(myurl, repo_type):
             # Test comps
             comps = libcomps.Comps()
             try:
-                ret = comps.fromxml_f(repo_info['group'])
+                # createrepo_c >= 1.0 also compresses the comps file
+                if repo_info['group'].endswith('xz'):
+                    xml_content = lzma.open(repo_info['group']).read()
+                    ret = comps.fromxml_str(xml_content.decode())
+                elif repo_info['group'].endswith('gz'):
+                    xml_content = gzip.open(repo_info['group']).read()
+                    ret = comps.fromxml_str(xml_content.decode())
+                elif repo_info['group'].endswith('zst'):
+                    xml_content = zstandard.open(repo_info['group']).read()
+                    ret = comps.fromxml_str(xml_content.decode())
+                elif repo_info['group'].endswith('bz2'):
+                    xml_content = bz2.open(repo_info['group']).read()
+                    ret = comps.fromxml_str(xml_content.decode())
+                else:
+                    ret = comps.fromxml_f(repo_info['group'])
             except Exception:
                 raise RepodataException('Comps file unable to be parsed')
             if len(comps.groups) < 1:

bodhi-server/pyproject.toml

@@ -98,8 +98,7 @@ Jinja2 = ">=2.11.3"
 Markdown = ">=3.3.6"
 munch = ">=2.5.0"
 koji = ">=1.27.1"
-#libcomps ="^0.1.18"
-libcomps ="*"
+libcomps ="^0.1.20"
 packaging = ">=21.3"
 prometheus-client = ">=0.13.1"
 psycopg2 = ">=2.8.6"
@@ -114,6 +113,7 @@ PyYAML = ">=5.4.1"
 requests = ">=2.25.1"
 SQLAlchemy = ">=1.4, <2.1"
 waitress = ">=1.4.4"
+zstandard = "^0.21"
 
 [tool.pytest.ini_options]
 addopts = "--cov-config .coveragerc --cov=bodhi --cov-report term --cov-report xml --cov-report html"

bodhi-server/tests/base.py

@@ -478,7 +478,7 @@ def __call__(self):
             raise
 
 
-def mkmetadatadir(path, updateinfo=None, comps=None, source=False):
+def mkmetadatadir(path, updateinfo=None, comps=None, source=False, compress_type='xz'):
     """
     Generate package metadata for a given directory.
 
@@ -524,7 +524,10 @@ def mkmetadatadir(path, updateinfo=None, comps=None, source=False):
         with open(updateinfo, 'w') as f:
             f.write(updateinfofile)
 
-    createrepo_command = ['createrepo_c', '--xz', '--database', '--quiet', path]
+    createrepo_command = ['createrepo_c', '--database', '--quiet', path]
+
+    if compress_type:
+        createrepo_command.insert(1, f'--compress-type={compress_type}')
 
     if not source:
         for arg in ('--deltas', 'comps.xml', '--groupfile'):

bodhi-server/tests/test_util.py

@@ -26,6 +26,7 @@
 
 from webob.multidict import MultiDict
 import bleach
+import createrepo_c
 import packaging
 import pkg_resources
 import pytest
@@ -404,13 +405,50 @@ def teardown_method(self, method):
         shutil.rmtree(self.tempdir)
         super().teardown_method(method)
 
-    def test_correct_yum_repo(self):
-        """No Exception should be raised if the repo is normal."""
+    def test_correct_yum_repo_with_xz_compress(self):
+        """No Exception should be raised if the repo is normal.
+
+        This is using default XZ compression.
+        """
         base.mkmetadatadir(self.tempdir)
 
         # No exception should be raised here.
         util.sanity_check_repodata(self.tempdir, repo_type='yum')
 
+    def test_correct_yum_repo_with_gz_compress(self):
+        """No Exception should be raised if the repo is normal.
+
+        This is using GZ compression.
+        """
+        base.mkmetadatadir(self.tempdir, compress_type='gz')
+
+        # No exception should be raised here.
+        util.sanity_check_repodata(self.tempdir, repo_type='yum')
+
+    def test_correct_yum_repo_with_bz2_compress(self):
+        """No Exception should be raised if the repo is normal.
+
+        This is using BZ2 compression.
+        """
+        base.mkmetadatadir(self.tempdir, compress_type='bz2')
+
+        # No exception should be raised here.
+        util.sanity_check_repodata(self.tempdir, repo_type='yum')
+
+    @pytest.mark.skipif(
+        pkg_resources.parse_version(createrepo_c.VERSION) < pkg_resources.parse_version('1.0.0'),
+        reason='ZSTD compression requires createrepo_c 1.0.0 or higher'
+    )
+    def test_correct_yum_repo_with_zstd_compress(self):
+        """No Exception should be raised if the repo is normal.
+
+        This is using ZSTD compression.
+        """
+        base.mkmetadatadir(self.tempdir, compress_type='zstd')
+
+        # No exception should be raised here.
+        util.sanity_check_repodata(self.tempdir, repo_type='yum')
+
     def test_invalid_repo_type(self):
         """A ValueError should be raised with invalid repo type."""
         with pytest.raises(ValueError) as excinfo:

devel/ansible/roles/bodhi/tasks/main.yml

@@ -72,6 +72,7 @@
       - python3-sqlalchemy
       - python3-sqlalchemy_schemadisplay
       - python3-webtest
+      - python3-zstandard
       - redhat-rpm-config
       - screen
       - skopeo

devel/ci/Dockerfile-f38

@@ -4,7 +4,7 @@ LABEL maintainer="Mattia Verga <[email protected]>"
 # RUN echo "fastestmirror=False" >> /etc/dnf/dnf.conf
 # RUN echo "zchunk = False" >> /etc/dnf/dnf.conf
 
-RUN dnf install -y \
+RUN dnf --best install -y \
     createrepo_c \
     fedora-messaging \
     findutils \
@@ -57,6 +57,7 @@ RUN dnf install -y \
     python3-webtest \
     python3-wheel \
     python3-yaml \
+    python3-zstandard \
     rpm-build \
     rpmdevtools \
     skopeo

devel/ci/Dockerfile-f39

@@ -4,7 +4,7 @@ LABEL maintainer="Mattia Verga <[email protected]>"
 # RUN echo "fastestmirror=False" >> /etc/dnf/dnf.conf
 # RUN echo "zchunk = False" >> /etc/dnf/dnf.conf
 
-RUN dnf install -y \
+RUN dnf --best install -y \
     createrepo_c \
     fedora-messaging \
     findutils \
@@ -57,6 +57,7 @@ RUN dnf install -y \
     python3-webtest \
     python3-wheel \
     python3-yaml \
+    python3-zstandard \
     rpm-build \
     rpmdevtools \
     skopeo

devel/ci/Dockerfile-rawhide

@@ -4,7 +4,7 @@ LABEL maintainer="Mattia Verga <[email protected]>"
 # RUN echo "fastestmirror=False" >> /etc/dnf/dnf.conf
 # RUN echo "zchunk = False" >> /etc/dnf/dnf.conf
 
-RUN dnf install -y \
+RUN dnf --best install -y \
     createrepo_c \
     fedora-messaging \
     findutils \
@@ -57,6 +57,7 @@ RUN dnf install -y \
     python3-webtest \
     python3-wheel \
     python3-yaml \
+    python3-zstandard \
     rpm-build \
     rpmdevtools \
     skopeo

news/PR5455.dependency

@@ -0,0 +1 @@
+libcomps >= 0.20 is required to correctly validate repodata created with createrepo_c >= 1.0. Bodhi can now support all compression method available in createrepo_c