Merge pull request #3772 from jlebon/pr/override-repo-client-prep5

.gitignore

@@ -111,4 +111,5 @@
 /tests/check/test-lib-introspection.sh.log
 /tests/check/test-lib-introspection.sh.test
 /tests/check/test-lib-introspection.sh.trs
+/tests/kola
 _libs

src/app/rpmostree-override-builtins.cxx

@@ -21,6 +21,7 @@
 #include "config.h"
 
 #include "rpmostree-container.h"
+#include "rpmostree-core.h"
 #include "rpmostree-libbuiltin.h"
 #include "rpmostree-override-builtins.h"
 
@@ -70,18 +71,88 @@ static GOptionEntry remove_option_entries[] = { { "replace", 0, 0, G_OPTION_ARG_
                                                   &opt_replace_pkgs, "Replace a package", "RPM" },
                                                 { NULL } };
 
+static gboolean
+sort_replacements (RPMOSTreeOSExperimental *osexperimental_proxy,
+                   const char *const *replacement_args, char ***out_local,
+                   GUnixFDList **out_fd_list, GCancellable *cancellable, GError **error)
+{
+  g_autoptr (GPtrArray) local = g_ptr_array_new_with_free_func (g_free);
+  g_autoptr (GPtrArray) freeze = g_ptr_array_new_with_free_func (g_free);
+  for (const char *const *it = replacement_args; it && *it; it++)
+    {
+      auto arg = *it;
+      if (rpmostreecxx::is_http_arg (arg) || rpmostreecxx::is_rpm_arg (arg))
+        {
+          g_ptr_array_add (local, (gpointer)g_strdup (arg));
+        }
+      // if the pkg is a valid nevra, then treat it as frozen
+      else if (opt_freeze || rpmostree_is_valid_nevra (arg))
+        {
+          g_ptr_array_add (freeze, (gpointer)g_strdup (arg));
+        }
+      else
+        {
+          glnx_throw (error, "Remote overrides are not supported yet on the CLI");
+        }
+    }
+
+  g_autoptr (GUnixFDList) fd_list = NULL;
+  if (freeze->len > 0)
+    {
+      g_ptr_array_add (freeze, NULL);
+
+      /* this is really just a D-Bus wrapper around `rpmostree_find_and_download_packages` */
+      if (!rpmostree_osexperimental_call_download_packages_sync (
+              osexperimental_proxy, (const char *const *)freeze->pdata, opt_from, NULL, &fd_list,
+              cancellable, error))
+        return FALSE;
+
+      int nfds;
+      const int *fds = g_unix_fd_list_peek_fds (fd_list, &nfds);
+      for (guint i = 0; i < nfds; i++)
+        g_ptr_array_add (local, g_strdup_printf ("file:///proc/self/fd/%d", fds[i]));
+    }
+
+  if (local->len > 0)
+    {
+      g_ptr_array_add (local, NULL);
+      *out_local = (char **)g_ptr_array_free (util::move_nullify (local), FALSE);
+    }
+  else
+    *out_local = NULL;
+
+  *out_fd_list = util::move_nullify (fd_list);
+  return TRUE;
+}
+
 static gboolean
 handle_override (RPMOSTreeSysroot *sysroot_proxy, RpmOstreeCommandInvocation *invocation,
                  const char *const *override_remove, const char *const *override_replace,
                  const char *const *override_reset, GCancellable *cancellable, GError **error)
 {
   glnx_unref_object RPMOSTreeOS *os_proxy = NULL;
-  RPMOSTreeOSExperimental *osexperimental_proxy = NULL;
-
+  glnx_unref_object RPMOSTreeOSExperimental *osexperimental_proxy = NULL;
   if (!rpmostree_load_os_proxies (sysroot_proxy, opt_osname, cancellable, &os_proxy,
                                   &osexperimental_proxy, error))
     return FALSE;
 
+  if (!opt_experimental && (opt_freeze || opt_from))
+    return glnx_throw (error, "Must specify --experimental to use --freeze or --from");
+
+  g_autoptr (GUnixFDList) fd_list = NULL; /* this is just used to own the fds */
+
+  /* By default, we assume all the args are local overrides. If --from is used, we have to sort them
+   * first to handle --freeze and (eventually) remote overrides. */
+  g_auto (GStrv) override_replace_local = NULL;
+  if (override_replace && opt_from)
+    {
+      if (!sort_replacements (osexperimental_proxy, override_replace, &override_replace_local,
+                              &fd_list, cancellable, error))
+        return FALSE;
+
+      override_replace = override_replace_local;
+    }
+
   /* Perform uninstalls offline; users don't expect the "auto-update" behaviour here. But
    * note we might still need to fetch pkgs in the local replacement case (e.g. the
    * replacing pkg has an additional out-of-tree dep). */
@@ -99,66 +170,7 @@ handle_override (RPMOSTreeSysroot *sysroot_proxy, RpmOstreeCommandInvocation *in
   g_autoptr (GVariant) options = g_variant_ref_sink (g_variant_dict_end (&dict));
 
   g_autoptr (GVariant) previous_deployment = rpmostree_os_dup_default_deployment (os_proxy);
-  g_autoptr (GUnixFDList) fetched_rpms_fds = NULL;
-  g_autoptr (GPtrArray) override_replace_final = NULL;
-
-  if (!opt_experimental && (opt_freeze || opt_from))
-    return glnx_throw (error, "Must specify --experimental to use --freeze or --from");
-
-  if (override_replace && opt_from)
-    {
-      override_replace_final = g_ptr_array_new_with_free_func (free);
-      g_autoptr (GPtrArray) queries = g_ptr_array_new ();
-
-      for (const char *const *it = override_replace; it && *it; it++)
-        {
-          auto pkg = *it;
 
-          if (rpmostreecxx::is_http_arg (pkg) || rpmostreecxx::is_rpm_arg (pkg))
-            {
-              g_ptr_array_add (override_replace_final, (gpointer)g_strdup (pkg));
-            }
-          // if the pkg is a valid nevra, then treat it as frozen
-          else if (opt_freeze || rpmostree_is_valid_nevra (pkg))
-            {
-              g_ptr_array_add (queries, (gpointer)pkg);
-            }
-          else
-            {
-              return glnx_throw (error, "CLI remote overrides not supported yet");
-            }
-        }
-      if (queries->len > 0)
-        {
-          g_ptr_array_add (queries, NULL);
-          g_autofree char *refresh_transaction_address = NULL;
-
-          if (!rpmostree_os_call_refresh_md_sync (os_proxy, options, &refresh_transaction_address,
-                                                  cancellable, error))
-            return FALSE;
-          if (!rpmostree_transaction_get_response_sync (
-                  sysroot_proxy, (const char *)refresh_transaction_address, cancellable, error))
-            return FALSE;
-          if (!rpmostree_osexperimental_call_download_packages_sync (
-                  osexperimental_proxy, (const char *const *)queries->pdata, opt_from, NULL,
-                  &fetched_rpms_fds, cancellable, error))
-            return FALSE;
-
-          const int nfds = fetched_rpms_fds ? g_unix_fd_list_get_length (fetched_rpms_fds) : 0;
-          g_assert_cmpuint (nfds, >, 0);
-
-          for (guint i = 0; i < nfds; i++)
-            {
-              gint fd = g_unix_fd_list_get (fetched_rpms_fds, i, error);
-              if (fd < 0)
-                return FALSE;
-              g_ptr_array_add (override_replace_final,
-                               g_strdup_printf ("file:///proc/self/fd/%d", fd));
-            }
-          g_ptr_array_add (override_replace_final, NULL);
-          override_replace = (const char *const *)override_replace_final->pdata;
-        }
-    }
   g_autofree char *transaction_address = NULL;
   if (!rpmostree_update_deployment (os_proxy, NULL,     /* set-refspec */
                                     NULL,               /* set-revision */

src/daemon/rpmostree-sysroot-upgrader.cxx

@@ -684,6 +684,7 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self, GCancellable *ca
   g_assert (self->rsack);
 
   auto local_replacements = rpmostree_origin_get_overrides_local_replace (self->computed_origin);
+  auto remote_replacements = rpmostree_origin_get_overrides_replace (self->computed_origin);
   g_autoptr (GPtrArray) inactive_replacements = g_ptr_array_new ();
 
   for (auto &nevra_v : local_replacements)
@@ -706,6 +707,17 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self, GCancellable *ca
         g_ptr_array_add (inactive_replacements, (gpointer)nevra);
     }
 
+  for (auto &ovr : remote_replacements)
+    {
+      for (auto &pkgname_s : ovr.packages)
+        {
+          const char *pkgname = pkgname_s.c_str ();
+          /* make inactive if it's missing */
+          if (!rpmostree_sack_has_pkgname (self->rsack->sack, pkgname))
+            g_ptr_array_add (inactive_replacements, (gpointer)pkgname);
+        }
+    }
+
   if (inactive_replacements->len > 0)
     {
       rpmostree_output_message ("Inactive base replacements:");
@@ -714,6 +726,7 @@ finalize_replacement_overrides (RpmOstreeSysrootUpgrader *self, GCancellable *ca
           const char *item = (const char *)inactive_replacements->pdata[i];
           rpmostree_output_message ("  %s", item);
           rpmostree_origin_remove_override_replace_local (self->computed_origin, item);
+          rpmostree_origin_remove_override_replace (self->computed_origin, item);
         }
     }
 
@@ -1153,6 +1166,7 @@ rpmostree_sysroot_upgrader_prep_layering (RpmOstreeSysrootUpgrader *self,
   /* Do a bit more work to see whether or not we have to do assembly */
   if (!load_base_rsack (self, cancellable, error))
     return FALSE;
+  /* XXX: initialize context earlier and lower this into the core instead */
   if (!finalize_overrides (self, cancellable, error))
     return FALSE;
   if (!finalize_overlays (self, cancellable, error))

src/daemon/rpmostreed-os-experimental.cxx

@@ -171,16 +171,12 @@ static gboolean
 prepare_download_pkgs_txn (const gchar *const *queries, const char *source,
                            GUnixFDList **out_fd_list, GError **error)
 {
-  GUnixFDList *fd_list = NULL;
-  g_autoptr (GCancellable) cancellable = g_cancellable_new ();
-
   if (!queries || !*queries)
     return glnx_throw (error, "No queries passed");
 
   OstreeSysroot *sysroot = rpmostreed_sysroot_get_root (rpmostreed_sysroot_get ());
   OstreeDeployment *booted_deployment = ostree_sysroot_get_booted_deployment (sysroot);
   const char *osname = ostree_deployment_get_osname (booted_deployment);
-  g_autoptr (GPtrArray) dnf_pkgs = g_ptr_array_new_with_free_func (g_object_unref);
 
   g_autoptr (OstreeDeployment) cfg_merge_deployment
       = ostree_sysroot_get_merge_deployment (sysroot, osname);
@@ -191,73 +187,14 @@ prepare_download_pkgs_txn (const gchar *const *queries, const char *source,
   g_autofree char *origin_deployment_root
       = rpmostree_get_deployment_root (sysroot, origin_merge_deployment);
 
-  OstreeRepo *repo = ostree_sysroot_repo (sysroot);
-  g_autoptr (RpmOstreeContext) ctx = rpmostree_context_new_client (repo);
-  rpmostree_context_set_dnf_caching (ctx, RPMOSTREE_CONTEXT_DNF_CACHE_FOREVER);
-  /* We could bypass rpmostree_context_setup() here and call dnf_context_setup() ourselves
-   * since we're not actually going to perform any installation. Though it does provide us
-   * with the right semantics for install/source_root. */
-
-  if (!rpmostree_context_setup (ctx, NULL, origin_deployment_root, cancellable, error))
-    return glnx_prefix_error (error, "Setting up dnf context");
-
-  /* point libdnf to our repos dir */
-  rpmostree_context_configure_from_deployment (ctx, sysroot, cfg_merge_deployment);
-
-  if (!rpmostree_context_download_metadata (ctx, DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB,
-                                            cancellable, error))
-    return glnx_prefix_error (error, "Downloading metadata");
-
-  DnfSack *sack = dnf_context_get_sack (rpmostree_context_get_dnf (ctx));
-  CXX_TRY_VAR (parsed_source, rpmostreecxx::parse_override_source (source), error);
-
-  if (parsed_source.kind == rpmostreecxx::OverrideReplacementType::Repo)
-    {
-      const char *source_name = parsed_source.name.c_str ();
-      for (const char *const *it = queries; it && *it; it++)
-        {
-          auto pkg_name = static_cast<const char *> (*it);
-          g_autoptr (GPtrArray) pkglist = NULL;
-          HyNevra nevra = NULL;
-          g_auto (HySubject) subject = hy_subject_create (pkg_name);
-          hy_autoquery HyQuery query = hy_subject_get_best_solution (
-              subject, sack, NULL, &nevra, FALSE, TRUE, TRUE, TRUE, FALSE);
-          hy_query_filter (query, HY_PKG_REPONAME, HY_EQ, source_name);
-          pkglist = hy_query_run (query);
-          if (!pkglist || pkglist->len == 0)
-            return glnx_throw (error, "No matches for \"%s\" in repo '%s'", pkg_name, source_name);
-
-          g_ptr_array_add (dnf_pkgs, g_object_ref (pkglist->pdata[0]));
-        }
-    }
-  /* Future source kinds go here */
-  else
-    return glnx_throw (error, "Unsupported source type");
-
-  rpmostree_set_repos_on_packages (rpmostree_context_get_dnf (ctx), dnf_pkgs);
-
-  if (!rpmostree_download_packages (dnf_pkgs, cancellable, error))
-    return glnx_prefix_error (error, "Downloading packages");
-
-  fd_list = g_unix_fd_list_new ();
-  for (unsigned int i = 0; i < dnf_pkgs->len; i++)
-    {
-      DnfPackage *pkg = static_cast<DnfPackage *> (dnf_pkgs->pdata[i]);
-      const gchar *path = dnf_package_get_filename (pkg);
-      gint fd = -1;
+  g_autofree char *cfg_deployment_root
+      = rpmostree_get_deployment_root (sysroot, cfg_merge_deployment);
 
-      if (!glnx_openat_rdonly (AT_FDCWD, path, TRUE, &fd, error))
-        return FALSE;
-
-      if (g_unix_fd_list_append (fd_list, fd, error) < 0)
-        return FALSE;
-
-      if (!rpmostree_pkg_is_local (pkg))
-        {
-          if (!glnx_unlinkat (AT_FDCWD, path, 0, error))
-            return FALSE;
-        }
-    }
+  g_autoptr (GUnixFDList) fd_list = NULL;
+  g_autoptr (GCancellable) cancellable = g_cancellable_new ();
+  if (!rpmostree_find_and_download_packages (queries, source, origin_deployment_root,
+                                             cfg_deployment_root, &fd_list, cancellable, error))
+    return FALSE;
 
   *out_fd_list = util::move_nullify (fd_list);
   return TRUE;

src/libpriv/rpmostree-core.cxx

@@ -2294,6 +2294,79 @@ rpmostree_download_packages (GPtrArray *packages, GCancellable *cancellable, GEr
   return TRUE;
 }
 
+gboolean
+rpmostree_find_and_download_packages (const char *const *packages, const char *source,
+                                      const char *source_root, const char *repo_root,
+                                      GUnixFDList **out_fd_list, GCancellable *cancellable,
+                                      GError **error)
+{
+  CXX_TRY_VAR (parsed_source, rpmostreecxx::parse_override_source (source), error);
+
+  g_autoptr (RpmOstreeContext) ctx = rpmostree_context_new_base (NULL);
+  if (!rpmostree_context_setup (ctx, NULL, source_root, cancellable, error))
+    return glnx_prefix_error (error, "Setting up dnf context");
+
+  g_autofree char *reposdir = g_build_filename (repo_root ?: source_root, "etc/yum.repos.d", NULL);
+  dnf_context_set_repo_dir (ctx->dnfctx, reposdir);
+
+  if (!rpmostree_context_download_metadata (ctx, DNF_CONTEXT_SETUP_SACK_FLAG_SKIP_RPMDB,
+                                            cancellable, error))
+    return glnx_prefix_error (error, "Downloading metadata");
+
+  g_autoptr (GPtrArray) pkgs = g_ptr_array_new_with_free_func (g_object_unref);
+  DnfSack *sack = dnf_context_get_sack (rpmostree_context_get_dnf (ctx));
+  switch (parsed_source.kind)
+    {
+    case rpmostreecxx::OverrideReplacementType::Repo:
+      {
+        const char *repo = parsed_source.name.c_str ();
+        for (const char *const *it = packages; it && *it; it++)
+          {
+            g_auto (HySubject) subject = hy_subject_create (static_cast<const char *> (*it));
+            HyNevra nevra = NULL;
+            hy_autoquery HyQuery query = hy_subject_get_best_solution (
+                subject, sack, NULL, &nevra, FALSE, TRUE, FALSE, FALSE, FALSE);
+            hy_query_filter (query, HY_PKG_REPONAME, HY_EQ, repo);
+            g_autoptr (GPtrArray) results = hy_query_run (query);
+            if (!results || results->len == 0)
+              return glnx_throw (error, "No matches for \"%s\" in repo '%s'", subject, repo);
+            g_ptr_array_add (pkgs, g_object_ref (results->pdata[0]));
+          }
+        break;
+      }
+    default:
+      return glnx_throw (error, "Unsupported source type used in '%s'", source);
+    }
+
+  rpmostree_set_repos_on_packages (rpmostree_context_get_dnf (ctx), pkgs);
+
+  if (!rpmostree_download_packages (pkgs, cancellable, error))
+    return glnx_prefix_error (error, "Downloading packages");
+
+  g_autoptr (GUnixFDList) fd_list = g_unix_fd_list_new ();
+  for (unsigned int i = 0; i < pkgs->len; i++)
+    {
+      DnfPackage *pkg = static_cast<DnfPackage *> (pkgs->pdata[i]);
+      const char *path = dnf_package_get_filename (pkg);
+
+      glnx_autofd int fd = -1;
+      if (!glnx_openat_rdonly (AT_FDCWD, path, TRUE, &fd, error))
+        return FALSE;
+
+      if (g_unix_fd_list_append (fd_list, fd, error) < 0)
+        return FALSE;
+
+      if (!rpmostree_pkg_is_local (pkg))
+        {
+          if (!glnx_unlinkat (AT_FDCWD, path, 0, error))
+            return FALSE;
+        }
+    }
+
+  *out_fd_list = util::move_nullify (fd_list);
+  return TRUE;
+}
+
 gboolean
 rpmostree_context_download (RpmOstreeContext *self, GCancellable *cancellable, GError **error)
 {