Verify kOps generated jobs in CI

config/jobs/kubernetes/kops/build_jobs.py

@@ -13,10 +13,18 @@
 # limitations under the License.
 
 # pylint: disable=line-too-long
+import argparse
+import difflib
+import filecmp
 import hashlib
-import math
 import json
+import math
+import os
+import pathlib
 import re
+import shutil
+import sys
+import tempfile
 import yaml
 import jinja2 # pylint: disable=import-error
 
@@ -28,6 +36,7 @@
     distros_ssh_user,
     k8s_version_info,
     should_skip_newer_k8s,
+    script_dir,
 )
 
 # These are job tab names of unsupported grid combinations
@@ -36,7 +45,7 @@
 
 image = "gcr.io/k8s-staging-test-infra/kubekins-e2e:v20241230-3006692a6f-master"
 
-loader = jinja2.FileSystemLoader(searchpath="./templates")
+loader = jinja2.FileSystemLoader(searchpath=os.path.join(script_dir, "templates"))
 
 # A helper function to construct the URLs to our marker files
 def marker_updown_green(kops_version):
@@ -2413,9 +2422,57 @@ def generate_presubmits_e2e():
     'kops-presubmits-scale.yaml': generate_presubmits_scale,
 }
 
-def main():
+def output_file(filename: pathlib.Path, contents: str, verify: bool) -> str:
+    """
+    Outputs one job configuration file.
+    Return list of errors.
+    """
+    tmp = tempfile.NamedTemporaryFile(
+        "w",
+        prefix=f"{filename}.",
+        delete=False,
+    )
+    with tmp:
+        tmp.write(contents)
+        tmp.close()
+
+        filepath = os.path.join(script_dir, filename)
+        if not os.path.exists(filepath) and verify:
+            os.unlink(tmp.name)
+            return f"Can't verify content: {filepath} doesn't exist"
+        if verify:
+            equal = filecmp.cmp(filepath, tmp.name, shallow=False)
+            if not equal:
+                with open(filepath) as f:
+                    diff = difflib.context_diff(
+                        str(f.read()).splitlines(keepends=True),
+                        contents.splitlines(keepends=True),
+                        fromfile=filepath,
+                        tofile=tmp.name,
+                    )
+                    sys.stdout.writelines(diff)
+                    return f"Generated content for {filename} differs from existing"
+        else:
+            shutil.move(tmp.name, filepath)
+    return ""
+
+def main(argv):
+    parser = argparse.ArgumentParser(
+        prog="kOps {}".format(os.path.basename(__file__)),
+        description="Generate kOps job configuration files from templates",
+        formatter_class=argparse.RawTextHelpFormatter,
+    )
+    parser.add_argument(
+        "--verify",
+        action="store_true",
+        help="Verify if generated files are the same as existing",
+    )
+    args = parser.parse_args(argv)
+
+    errors = []
     for filename, generate_func in periodics_files.items():
-        print(f"Generating {filename}")
+        if not args.verify:
+            print(f"Generating {filename}")
         output = []
         runs_per_week = 0
         job_count = 0
@@ -2426,10 +2483,11 @@ def main():
         output.insert(0, "# Test jobs generated by build_jobs.py (do not manually edit)\n")
         output.insert(1, f"# {job_count} jobs, total of {runs_per_week} runs per week\n")
         output.insert(2, "periodics:\n")
-        with open(filename, 'w') as fd:
-            fd.write(''.join(output))
+        errors.append(output_file(filename, ''.join(output), args.verify))
+
     for filename, generate_func in presubmits_files.items():
-        print(f"Generating {filename}")
+        if not args.verify:
+            print(f"Generating {filename}")
         output = []
         job_count = 0
         for res in generate_func():
@@ -2439,8 +2497,15 @@ def main():
         output.insert(1, f"# {job_count} jobs\n")
         output.insert(2, "presubmits:\n")
         output.insert(3, "  kubernetes/kops:\n")
-        with open(filename, 'w') as fd:
-            fd.write(''.join(output))
+        errors.append(output_file(filename, ''.join(output), args.verify))
+
+    errors = list(filter(None, errors))
+    if len(errors) > 0:
+        print(f"Errors: {len(errors)}")
+        for error in errors:
+            print(error)
+        return 1
+    return 0
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main(sys.argv[1:]))

config/jobs/kubernetes/kops/helpers.py

@@ -12,10 +12,9 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import os
 import zlib
 
-import boto3 # pylint: disable=import-error
-
 # We support rapid focus on a few tests of high concern
 # This should be used for temporary tests we are evaluating,
 # and ideally linked to a bug, and removed once the bug is fixed
@@ -25,6 +24,8 @@
 run_daily = [
 ]
 
+script_dir = os.path.dirname(os.path.realpath(__file__))
+
 def simple_hash(s):
     # & 0xffffffff avoids python2/python3 compatibility
     return zlib.crc32(s.encode()) & 0xffffffff
@@ -112,7 +113,7 @@ def create_args(kops_channel, networking, extra_flags, kops_image):
 # The pin file contains a list of key=value pairs, that holds images we want to pin.
 # This enables us to use the latest image without fetching them from AWS every time.
 def pinned_file():
-    return 'pinned.list'
+    return os.path.join(script_dir, 'pinned.list')
 
 # get_pinned returns the pinned value for the given key, or None if the key is not found
 def get_pinned(key):
@@ -144,6 +145,9 @@ def latest_aws_image(owner, name, arch='x86_64'):
     image = get_pinned(pin)
     if image:
         return image
+
+    import boto3 # pylint: disable=import-error, import-outside-toplevel
+
     client = boto3.client('ec2', region_name='us-east-1')
     response = client.describe_images(
         Owners=[owner],

hack/make-rules/update/generated-jobs.sh

@@ -24,3 +24,7 @@ hack/run-in-python-container.sh \
 echo "Generate jobs"
 hack/run-in-python-container.sh \
     python3 hack/generate-jobs.py config/jobs/kubernetes/sig-node/*.conf
+
+echo "Generate kOps jobs"
+hack/run-in-python-container.sh \
+    python3 config/jobs/kubernetes/kops/build_jobs.py

hack/make-rules/verify/generated-jobs.sh

@@ -24,3 +24,7 @@ hack/run-in-python-container.sh \
 echo "Verifying generated jobs"
 hack/run-in-python-container.sh \
     python3 hack/generate-jobs.py config/jobs/**/*.generate.conf --verify
+
+echo "Verifying generated kOps jobs"
+hack/run-in-python-container.sh \
+    python3 config/jobs/kubernetes/kops/build_jobs.py --verify