Support Flatpak preinstallation as part of a DNF install

anaconda.spec.in

@@ -111,6 +111,7 @@ Requires: python3-pwquality
 Requires: python3-systemd
 Requires: python3-productmd
 Requires: python3-dasbus >= %{dasbusver}
+Requires: flatpak
 Requires: flatpak-libs
 %if %{defined rhel} && %{undefined centos}
 Requires: subscription-manager >= %{subscriptionmanagerver}

configure.ac

@@ -164,6 +164,7 @@ AC_CONFIG_FILES([Makefile
                  pyanaconda/modules/payloads/Makefile
                  pyanaconda/modules/payloads/payload/Makefile
                  pyanaconda/modules/payloads/payload/dnf/Makefile
+                 pyanaconda/modules/payloads/payload/flatpak/Makefile
                  pyanaconda/modules/payloads/payload/live_os/Makefile
                  pyanaconda/modules/payloads/payload/live_image/Makefile
                  pyanaconda/modules/payloads/payload/rpm_ostree/Makefile

pyanaconda/core/constants.py

@@ -394,6 +394,7 @@ class DisplayModes(Enum):
 
 # Types of the payload.
 PAYLOAD_TYPE_DNF = "DNF"
+PAYLOAD_TYPE_FLATPAK = "FLATPAK"
 PAYLOAD_TYPE_LIVE_OS = "LIVE_OS"
 PAYLOAD_TYPE_LIVE_IMAGE = "LIVE_IMAGE"
 PAYLOAD_TYPE_RPM_OSTREE = "RPM_OSTREE"

pyanaconda/modules/common/constants/interfaces.py

@@ -74,6 +74,11 @@
     basename="DNF"
 )
 
+PAYLOAD_FLATPAK = DBusInterfaceIdentifier(
+    namespace=PAYLOAD_NAMESPACE,
+    basename="FLATPAK"
+)
+
 PAYLOAD_LIVE_IMAGE = DBusInterfaceIdentifier(
     namespace=PAYLOAD_NAMESPACE,
     basename="LiveImage"

pyanaconda/modules/payloads/base/utils.py

@@ -17,12 +17,258 @@
 # License and may only be used or replicated with the express permission of
 # Red Hat, Inc.
 #
+import os
+from functools import partial
+
+from blivet.size import Size
+
 from pyanaconda.anaconda_loggers import get_module_logger
-from pyanaconda.core.payload import rpm_version_key
+from pyanaconda.core.configuration.anaconda import conf
+from pyanaconda.core.constants import NETWORK_CONNECTION_TIMEOUT, USER_AGENT
+from pyanaconda.core.path import join_paths
+from pyanaconda.core.payload import ProxyString, ProxyStringError, rpm_version_key
+from pyanaconda.core.util import execWithCapture
+from pyanaconda.modules.common.constants.objects import DEVICE_TREE
+from pyanaconda.modules.common.constants.services import STORAGE
+from pyanaconda.modules.common.structures.payload import RepoConfigurationData
 
 log = get_module_logger(__name__)
 
 
 def sort_kernel_version_list(kernel_version_list):
     """Sort the given kernel version list."""
     kernel_version_list.sort(key=rpm_version_key)
+
+
+def get_downloader_for_repo_configuration(session, data: RepoConfigurationData):
+    """Get a configured session.get method.
+
+    :return: a partial function
+    """
+    # Prepare the SSL configuration.
+    ssl_enabled = conf.payload.verify_ssl and data.ssl_verification_enabled
+
+    # ssl_verify can be:
+    #   - the path to a cert file
+    #   - True, to use the system's certificates
+    #   - False, to not verify
+    ssl_verify = data.ssl_configuration.ca_cert_path or ssl_enabled
+
+    # ssl_cert can be:
+    #   - a tuple of paths to a client cert file and a client key file
+    #   - None
+    ssl_client_cert = data.ssl_configuration.client_cert_path or None
+    ssl_client_key = data.ssl_configuration.client_key_path or None
+    ssl_cert = (ssl_client_cert, ssl_client_key) if ssl_client_cert else None
+
+    # Prepare the proxy configuration.
+    proxy_url = data.proxy or None
+    proxies = {}
+
+    if proxy_url:
+        try:
+            proxy = ProxyString(proxy_url)
+            proxies = {
+                "http": proxy.url,
+                "https": proxy.url,
+                "ftp": proxy.url
+            }
+        except ProxyStringError as e:
+            log.debug("Failed to parse the proxy '%s': %s", proxy_url, e)
+
+    # Prepare headers.
+    headers = {"user-agent": USER_AGENT}
+
+    # Return a partial function.
+    return partial(
+        session.get,
+        headers=headers,
+        proxies=proxies,
+        verify=ssl_verify,
+        cert=ssl_cert,
+        timeout=NETWORK_CONNECTION_TIMEOUT
+    )
+
+
+def get_free_space_map(current=True, scheduled=False):
+    """Get the available file system disk space.
+
+    :param bool current: use information about current mount points
+    :param bool scheduled: use information about scheduled mount points
+    :return: a dictionary of mount points and their available space
+    """
+    mount_points = {}
+
+    if scheduled:
+        mount_points.update(_get_scheduled_free_space_map())
+
+    if current:
+        mount_points.update(_get_current_free_space_map())
+
+    return mount_points
+
+
+def _get_current_free_space_map():
+    """Get the available file system disk space of the current system.
+
+    :return: a dictionary of mount points and their available space
+    """
+    mapping = {}
+
+    # Generate the dictionary of mount points and sizes.
+    output = execWithCapture('df', ['--output=target,avail'])
+    lines = output.rstrip().splitlines()
+
+    for line in lines:
+        key, val = line.rsplit(maxsplit=1)
+
+        if not key.startswith('/'):
+            continue
+
+        mapping[key] = Size(int(val) * 1024)
+
+    # Add /var/tmp/ if this is a directory or image installation.
+    if not conf.target.is_hardware:
+        var_tmp = os.statvfs("/var/tmp")
+        mapping["/var/tmp"] = Size(var_tmp.f_frsize * var_tmp.f_bfree)
+
+    return mapping
+
+
+def _get_scheduled_free_space_map():
+    """Get the available file system disk space of the scheduled system.
+
+    :return: a dictionary of mount points and their available space
+    """
+    device_tree = STORAGE.get_proxy(DEVICE_TREE)
+    mount_points = {}
+
+    for mount_point in device_tree.GetMountPoints():
+        # we can ignore swap
+        if not mount_point.startswith('/'):
+            continue
+
+        free_space = Size(
+            device_tree.GetFileSystemFreeSpace([mount_point])
+        )
+        mount_point = os.path.normpath(
+            conf.target.system_root + mount_point
+        )
+        mount_points[mount_point] = free_space
+
+    return mount_points
+
+
+def _pick_mount_points(mount_points, download_size, install_size):
+    """Pick mount points for the package installation.
+
+    :return: a set of sufficient mount points
+    """
+    suitable = {
+        '/var/tmp',
+        conf.target.system_root,
+        join_paths(conf.target.system_root, 'home'),
+        join_paths(conf.target.system_root, 'tmp'),
+        join_paths(conf.target.system_root, 'var'),
+    }
+
+    sufficient = set()
+
+    for mount_point, size in mount_points.items():
+        # Ignore mount points that are not suitable.
+        if mount_point not in suitable:
+            continue
+
+        if size >= (download_size + install_size):
+            log.debug("Considering %s (%s) for download and install.", mount_point, size)
+            sufficient.add(mount_point)
+
+        elif size >= download_size and not mount_point.startswith(conf.target.system_root):
+            log.debug("Considering %s (%s) for download.", mount_point, size)
+            sufficient.add(mount_point)
+
+    return sufficient
+
+
+def _get_biggest_mount_point(mount_points, sufficient):
+    """Get the biggest sufficient mount point.
+
+    :return: a mount point or None
+    """
+    return max(sufficient, default=None, key=mount_points.get)
+
+
+def pick_download_location(download_size, install_size, cache_dir_suffix):
+    """Pick a download location
+
+    :param dnf_manager: the DNF manager
+    :return: a path to the download location
+    """
+    mount_points = get_free_space_map()
+
+    # Try to find mount points that are sufficient for download and install.
+    sufficient = _pick_mount_points(
+        mount_points,
+        download_size,
+        install_size
+    )
+
+    # Or find mount points that are sufficient only for download.
+    if not sufficient:
+        sufficient = _pick_mount_points(
+            mount_points,
+            download_size,
+            install_size=0
+        )
+
+    # Ignore the system root if there are other mount points.
+    if len(sufficient) > 1:
+        sufficient.discard(conf.target.system_root)
+
+    if not sufficient:
+        raise RuntimeError(
+            "Not enough disk space to download the "
+            "packages; size {}.".format(download_size)
+        )
+
+    # Choose the biggest sufficient mount point.
+    mount_point = _get_biggest_mount_point(mount_points, sufficient)
+
+    log.info("Mount point %s picked as download location", mount_point)
+    location = join_paths(mount_point, cache_dir_suffix)
+
+    return location
+
+
+def calculate_required_space(download_size, installation_size):
+    """Calculate the space required for the installation.
+
+    This takes into account whether the download location is part of the installed
+    system or not.
+
+    :param Size download_size: the download size
+    :param Size installation_size: the installation size
+    :return Size: the required space
+    """
+    mount_points = get_free_space_map(scheduled=True)
+
+    # Find sufficient mount points.
+    sufficient = _pick_mount_points(
+        mount_points,
+        download_size,
+        installation_size
+    )
+
+    # Choose the biggest sufficient mount point.
+    mount_point = _get_biggest_mount_point(mount_points, sufficient)
+
+    if not mount_point or mount_point.startswith(conf.target.system_root):
+        log.debug("The install and download space is required.")
+        required_space = installation_size + download_size
+    else:
+        log.debug("Use the %s mount point for the %s download.", mount_point, download_size)
+        log.debug("Only the install space is required.")
+        required_space = installation_size
+
+    log.debug("The package installation requires %s.", required_space)
+    return required_space

pyanaconda/modules/payloads/constants.py

@@ -20,6 +20,7 @@
 
 from pyanaconda.core.constants import (
     PAYLOAD_TYPE_DNF,
+    PAYLOAD_TYPE_FLATPAK,
     PAYLOAD_TYPE_LIVE_IMAGE,
     PAYLOAD_TYPE_LIVE_OS,
     PAYLOAD_TYPE_RPM_OSTREE,
@@ -51,6 +52,7 @@
 class PayloadType(Enum):
     """Type of the payload."""
     DNF = PAYLOAD_TYPE_DNF
+    FLATPAK = PAYLOAD_TYPE_FLATPAK
     LIVE_OS = PAYLOAD_TYPE_LIVE_OS
     LIVE_IMAGE = PAYLOAD_TYPE_LIVE_IMAGE
     RPM_OSTREE = PAYLOAD_TYPE_RPM_OSTREE

pyanaconda/modules/payloads/payload/Makefile.am

@@ -14,7 +14,7 @@
 # You should have received a copy of the GNU Lesser General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-SUBDIRS = dnf live_os live_image rpm_ostree
+SUBDIRS = dnf flatpak live_os live_image rpm_ostree
 
 pkgpyexecdir = $(pyexecdir)/py$(PACKAGE_NAME)
 dnf_moduledir = $(pkgpyexecdir)/modules/payloads/payload

pyanaconda/modules/payloads/payload/dnf/dnf.py

@@ -24,6 +24,7 @@
     PackagesConfigurationData,
     PackagesSelectionData,
 )
+from pyanaconda.modules.payloads.base.utils import calculate_required_space
 from pyanaconda.modules.payloads.constants import PayloadType, SourceType
 from pyanaconda.modules.payloads.kickstart import (
     convert_ks_data_to_packages_configuration,
@@ -54,7 +55,6 @@
 )
 from pyanaconda.modules.payloads.payload.dnf.tear_down import ResetDNFManagerTask
 from pyanaconda.modules.payloads.payload.dnf.utils import (
-    calculate_required_space,
     collect_installation_devices,
     protect_installation_devices,
 )
@@ -388,9 +388,20 @@ def calculate_required_space(self):
         :return: required size in bytes
         :rtype: int
         """
-        required_space = calculate_required_space(self.dnf_manager)
+        required_space = calculate_required_space(self._dnf_manager.get_download_size(),
+                                                  self._dnf_manager.get_installation_size())
         return required_space.get_bytes()
 
+    def needs_flatpak_side_payload(self):
+        return True
+
+    def get_flatpak_refs(self):
+        """Get the list of Flatpak refs to install.
+
+        :return: list of Flatpak refs
+        """
+        return self._dnf_manager.get_flatpak_refs()
+
     def get_repo_configurations(self):
         """Get RepoConfiguration structures for all sources.
 

pyanaconda/modules/payloads/payload/dnf/dnf_manager.py

@@ -18,6 +18,7 @@
 # Red Hat, Inc.
 #
 import multiprocessing
+import re
 import shutil
 import threading
 import traceback
@@ -598,6 +599,20 @@ def resolve_selection(self):
         log.info("The software selection has been resolved (%d packages selected).",
                  len(self._base.transaction))
 
+    def get_flatpak_refs(self):
+        """Determine what Flatpaks need to be preinstalled based on resolved transaction"""
+        if self._base.transaction is None:
+            return []
+
+        refs = []
+        for tsi in self._base.transaction:
+            for provide in tsi.pkg.provides:
+                m = re.match(r"^flatpak-preinstall\((.*)\)$", str(provide))
+                if m:
+                    refs.append(m.group(1))
+
+        return refs
+
     def clear_selection(self):
         """Clear the software selection."""
         self._base.reset(goal=True)