Merge upgrade and preupgrade script, add sync script

.github/workflows/check-scripts-updates.yml

@@ -20,8 +20,8 @@ jobs:
 
         changed_files=$(git diff --name-only "$BASE_COMMIT" "$HEAD_COMMIT")
 
-        # Check if any files in scripts/ were updated (excluding wrap_scripts_in_yaml.py)
-        scripts_updated=$(echo "$changed_files" | grep -v 'wrap_scripts_in_yaml.py' | grep 'scripts/' || true)
+        # Check if any files in scripts/ were updated
+        scripts_updated=$(echo "$changed_files" | grep 'scripts/' || true)
         echo "Scripts updated: $scripts_updated"
 
         # Check if any files in playbooks/ were updated

.pre-commit-config.yaml

@@ -29,9 +29,10 @@ repos:
             "-sn", # Don't display the score
             "--rcfile=.pylintrc", # Link to your config file
           ]
-      - id: wrap-scripts-in-yaml
-        name: wrap-scripts-in-yaml
-        entry: python scripts/wrap_scripts_in_yaml.py
+      - id: sync-scripts-to-yaml
+        name: sync-scripts-to-yaml
+        entry: python misc/sync_scripts_to_yaml.py
+        args: ["--target", "repo"]
         language: python
         files: scripts/.*\.py$
 

misc/sync_scripts.py

@@ -0,0 +1,93 @@
+import os
+import argparse
+import ruamel.yaml
+
+# Scripts located in this project
+SCRIPT_PATH = "scripts/leapp_script.py"
+
+REPO_PRE_UPGRADE_YAML_PATH = os.path.join(".", "playbooks/leapp_preupgrade_script.yaml")
+REPO_UPGRADE_YAML_PATH = os.path.join(".", "playbooks/leapp_upgrade_script.yaml")
+
+WORKER_PRE_UPGRADE_YAML_PATH = os.path.join(
+    "..", "rhc-worker-script/development/nginx/data/leapp_preupgrade.yml"
+)
+WORKER_UPGRADE_YAML_PATH = os.path.join(
+    "..", "rhc-worker-script/development/nginx/data/leapp_upgrade.yml"
+)
+
+DEFAULT_YAML_ENVELOPE = """
+- name: LEAPP
+  vars:
+    insights_signature: |
+      ascii_armored gpg signature
+    insights_signature_exclude: /vars/insights_signature,/vars/content_vars
+    interpreter: /usr/bin/python
+    content: |
+      placeholder
+    content_vars:
+      # variables that will be handed to the script as environment vars
+      # will be prefixed with RHC_WORKER_*
+      LEAPP_SCRIPT_TYPE: type
+"""
+
+
+def _get_updated_yaml_content(yaml_path, script_path):
+    if not os.path.exists(yaml_path):
+        yaml = ruamel.yaml.YAML()
+        config = yaml.load(DEFAULT_YAML_ENVELOPE)
+        mapping = 2
+        offset = 0
+    else:
+        config, mapping, offset = ruamel.yaml.util.load_yaml_guess_indent(
+            open(yaml_path)
+        )
+        print(mapping, offset)
+
+    with open(script_path) as script:
+        content = script.read()
+
+    script_type = "PREUPGRADE" if "preupgrade" in yaml_path else "UPGRADE"
+    config[0]["name"] = "Leapp %s for rhc-worker-script" % script_type.title()
+    config[0]["vars"]["content"] = content
+    config[0]["vars"]["content_vars"]["LEAPP_SCRIPT_TYPE"] = script_type
+    return config, mapping, offset
+
+
+def _write_content(config, path, mapping=None, offset=None):
+    yaml = ruamel.yaml.YAML()
+    if mapping and offset:
+        yaml.indent(mapping=mapping, sequence=mapping, offset=offset)
+    with open(path, "w") as handler:
+        yaml.dump(config, handler)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--target",
+        choices=["repo", "worker"],
+        help="Target to sync scripts to",
+        default="worker",
+    )
+    args = parser.parse_args()
+
+    if args.target == "repo":
+        print("Syncing scripts to repo")
+        pre_upgrade_path = REPO_PRE_UPGRADE_YAML_PATH
+        upgrade_path = REPO_UPGRADE_YAML_PATH
+
+    elif args.target == "worker":
+        print("Syncing scripts to worker")
+        pre_upgrade_path = WORKER_PRE_UPGRADE_YAML_PATH
+        upgrade_path = WORKER_UPGRADE_YAML_PATH
+
+    config, mapping, offset = _get_updated_yaml_content(pre_upgrade_path, SCRIPT_PATH)
+    print("Writing new content to %s" % pre_upgrade_path)
+    _write_content(config, pre_upgrade_path, mapping, offset)
+    config, mapping, offset = _get_updated_yaml_content(upgrade_path, SCRIPT_PATH)
+    print("Writing new content to %s" % upgrade_path)
+    _write_content(config, upgrade_path, mapping, offset)
+
+
+if __name__ == "__main__":
+    main()

playbooks/leapp_preupgrade_script.yaml

@@ -1,17 +1,22 @@
-- name: Leapp pre-upgrade for rhc-worker-script
+- name: Leapp Preupgrade for rhc-worker-script
   vars:
-    insights_signature: !!binary |
-      needs signature
-    insights_signature_exclude: "/vars/insights_signature"
+    insights_signature: |
+      ascii_armored gpg signature
+    insights_signature_exclude: /vars/insights_signature,/vars/content_vars
     interpreter: /usr/bin/python
     content: |
       import json
       import os
       import subprocess
 
-
+      # SCRIPT_TYPE is either 'PREUPGRADE' or 'UPGRADE'
+      # Value is set in signed yaml envelope in content_vars (RHC_WORKER_LEAPP_SCRIPT_TYPE)
+      SCRIPT_TYPE = os.environ.get("RHC_WORKER_LEAPP_SCRIPT_TYPE", "None")
+      IS_UPGRADE = SCRIPT_TYPE == "UPGRADE"
+      IS_PREUPGRADE = SCRIPT_TYPE == "PREUPGRADE"
       JSON_REPORT_PATH = "/var/log/leapp/leapp-report.json"
       TXT_REPORT_PATH = "/var/log/leapp/leapp-report.txt"
+      REBOOT_GUIDANCE_MESSAGE = "A reboot is required to continue. Please reboot your system."
 
       # Based on https://github.com/oamg/leapp/blob/master/report-schema-v110.json#L211
       STATUS_CODE = {
@@ -103,6 +108,7 @@
 
 
       def is_non_eligible_releases(release):
+          """Check if the release is eligible for upgrade or preupgrade."""
           print("Exit if not RHEL 7 or RHEL 8 ...")
           major_version, _ = release.split(".") if release is not None else (None, None)
           return release is None or major_version not in ["7", "8"]
@@ -220,15 +226,18 @@
 
 
       def setup_leapp(version):
-          print("Installing leapp ...")
           leapp_install_command, rhel_rhui_packages = _get_leapp_command_and_packages(version)
-          output, returncode = run_subprocess(leapp_install_command)
-          if returncode:
-              raise ProcessError(
-                  message="Installation of leapp failed",
-                  report="Installation of leapp failed with code '%s' and output: %s."
-                  % (returncode, output.rstrip("\n")),
-              )
+          if _check_if_package_installed('leapp-upgrade'):
+              print("'leapp-upgrade' already installed, skipping ...")
+          else:
+              print("Installing leapp ...")
+              output, returncode = run_subprocess(leapp_install_command)
+              if returncode:
+                  raise ProcessError(
+                      message="Installation of leapp failed",
+                      report="Installation of leapp failed with code '%s' and output: %s."
+                      % (returncode, output.rstrip("\n")),
+                  )
 
           print("Check installed rhui packages ...")
           for pkg in rhel_rhui_packages:
@@ -252,7 +261,11 @@
               )
 
           if rhui_installed and not rhsm_repo_check_fail:
-              print("RHUI packages detected, adding --no-rhsm flag to preupgrade command")
+              print(
+                  "RHUI packages detected, adding --no-rhsm flag to {} command".format(
+                      SCRIPT_TYPE.title()
+                  )
+              )
               command.append("--no-rhsm")
               return True
           return False
@@ -274,7 +287,7 @@
 
 
       def remove_previous_reports():
-          print("Removing previous preupgrade reports at /var/log/leapp/leapp-report.* ...")
+          print("Removing previous leapp reports at /var/log/leapp/leapp-report.* ...")
 
           if os.path.exists(JSON_REPORT_PATH):
               os.remove(JSON_REPORT_PATH)
@@ -283,17 +296,11 @@
               os.remove(TXT_REPORT_PATH)
 
 
-      def execute_preupgrade(command):
-          print("Executing preupgrade ...")
-          _, _ = run_subprocess(command)
+      def execute_operation(command):
+          print("Executing {} ...".format(SCRIPT_TYPE.title()))
+          output, _ = run_subprocess(command)
 
-          # NOTE: we do not care about returncode because non-null always means actor error (or leapp error)
-          # if returncode:
-          #     print(
-          #         "The process leapp exited with code '%s' and output: %s\n"
-          #         % (returncode, output)
-          #     )
-          #     raise ProcessError(message="Leapp exited with code '%s'." % returncode)
+          return output
 
 
       def _find_highest_report_level(entries):
@@ -312,8 +319,8 @@
           return STATUS_CODE_NAME_MAP[valid_action_levels[0]]
 
 
-      def parse_results(output):
-          print("Processing preupgrade results ...")
+      def parse_results(output, reboot_required=False):
+          print("Processing {} results ...".format(SCRIPT_TYPE.title()))
 
           report_json = "Not found"
           message = "Can't open json report at " + JSON_REPORT_PATH
@@ -325,16 +332,28 @@
               with open(JSON_REPORT_PATH, mode="r") as handler:
                   report_json = json.load(handler)
 
-              # NOTE: with newer schema we will need to parse groups instead of flags
               report_entries = report_json.get("entries", [])
+
+              error_count = len(
+                  [entry for entry in report_entries if "error" in entry.get("groups")]
+              )
               inhibitor_count = len(
-                  [entry for entry in report_entries if "inhibitor" in entry.get("flags")]
+                  [entry for entry in report_entries if "inhibitor" in entry.get("groups")]
               )
-              message = "Your system has %s inhibitors out of %s potential problems." % (
-                  inhibitor_count,
-                  len(report_entries),
+              message = (
+                  "Your system has %s error%s and %s inhibitor%s out of %s potential problem%s."
+                  % (
+                      error_count,
+                      "" if error_count == 1 else "s",
+                      inhibitor_count,
+                      "" if inhibitor_count == 1 else "s",
+                      len(report_entries),
+                      "" if len(report_entries) == 1 else "s",
+                  )
               )
-              alert = inhibitor_count > 0
+              if reboot_required:
+                  message += " System is ready to be upgraded. Rebooting system in 1 minute."
+              alert = inhibitor_count > 0 or error_count > 0
               status = (
                   _find_highest_report_level(report_entries)
                   if len(report_entries) > 0
@@ -355,24 +374,36 @@
           output.report = report_txt
 
 
-      def update_insights_inventory():
+      def update_insights_inventory(output):
           """Call insights-client to update insights inventory."""
           print("Updating system status in Red Hat Insights.")
-          output, returncode = run_subprocess(cmd=["/usr/bin/insights-client"])
+          _, returncode = run_subprocess(cmd=["/usr/bin/insights-client"])
 
           if returncode:
-              raise ProcessError(
-                  message="Failed to update Insights Inventory by registering the system again. See output the following output: %s"
-                  % output,
-                  report="insights-client execution exited with code '%s'." % returncode,
-              )
+              print("System registration failed with exit code %s." % returncode)
+              output.message += " Failed to update Insights Inventory."
+              output.alert = True
+              return
 
           print("System registered with insights-client successfully.")
 
 
+      def reboot_system():
+          print("Rebooting system in 1 minute.")
+          run_subprocess(["/usr/sbin/shutdown", "-r", "1"], wait=False)
+
+
       def main():
           try:
-              # Exit if not RHEL 7 or 8
+              # Exit if invalid value for SCRIPT_TYPE
+              if SCRIPT_TYPE not in ["PREUPGRADE", "UPGRADE"]:
+                  raise ProcessError(
+                      message="Allowed values for RHC_WORKER_LEAPP_SCRIPT_TYPE are 'PREUPGRADE' and 'UPGRADE'.",
+                      report="Exiting because RHC_WORKER_LEAPP_SCRIPT_TYPE='%s'"
+                      % SCRIPT_TYPE,
+                  )
+
+              # Exit if not eligible release
               dist, version = get_rhel_version()
               if dist != "rhel" or is_non_eligible_releases(version):
                   raise ProcessError(
@@ -382,19 +413,24 @@
                   )
 
               output = OutputCollector()
-              preupgrade_command = ["/usr/bin/leapp", "preupgrade", "--report-schema=1.1.0"]
+              preupgrade_command = ["/usr/bin/leapp", "preupgrade", "--report-schema=1.2.0"]
+              upgrade_command = ["/usr/bin/leapp", "upgrade", "--report-schema=1.2.0"]
+              operation_command = preupgrade_command if IS_PREUPGRADE else upgrade_command
               rhui_pkgs = setup_leapp(version)
 
               # Check for RHUI PKGs
-              use_no_rhsm = should_use_no_rhsm_check(len(rhui_pkgs) > 1, preupgrade_command)
+              use_no_rhsm = should_use_no_rhsm_check(len(rhui_pkgs) > 1, operation_command)
               if use_no_rhsm:
                   install_leapp_pkg_corresponding_to_installed_rhui(rhui_pkgs)
 
               remove_previous_reports()
-              execute_preupgrade(preupgrade_command)
-              parse_results(output)
-              update_insights_inventory()
-              print("Pre-upgrade successfully executed.")
+              leapp_output = execute_operation(operation_command)
+              reboot_required = REBOOT_GUIDANCE_MESSAGE in leapp_output
+              parse_results(output, reboot_required)
+              update_insights_inventory(output)
+              print("Operation {} finished successfully.".format(SCRIPT_TYPE.title()))
+              if reboot_required:
+                  reboot_system()
           except ProcessError as exception:
               print(exception.report)
               output = OutputCollector(
@@ -422,3 +458,6 @@
       if __name__ == "__main__":
           main()
     content_vars:
+      # variables that will be handed to the script as environment vars
+      # will be prefixed with RHC_WORKER_*
+      LEAPP_SCRIPT_TYPE: PREUPGRADE