From b144c5285df37a00f27b7215c40fe823abb71c1e Mon Sep 17 00:00:00 2001 From: Greg Rose Date: Thu, 9 Jan 2025 10:40:39 -0800 Subject: [PATCH] github actions: Use python PR check script jira LE-2214 Obsoletes the old ruby PR check script --- .github/workflows/process-git-request.py | 190 +++++++++++++++++++++ .github/workflows/process-git-request.rb | 140 --------------- .github/workflows/process-pull-request.yml | 32 ++-- 3 files changed, 202 insertions(+), 160 deletions(-) create mode 100644 .github/workflows/process-git-request.py delete mode 100644 .github/workflows/process-git-request.rb diff --git a/.github/workflows/process-git-request.py b/.github/workflows/process-git-request.py new file mode 100644 index 000000000000..87b16ebd29cf --- /dev/null +++ b/.github/workflows/process-git-request.py @@ -0,0 +1,190 @@ +#!/usr/bin/env python3 + +import sys +import subprocess +import os +import re +import git + +def log_commits_between_branches(repo_path, from_branch, to_branch): + repo = git.Repo(repo_path) + + # Get the common ancestor of the two branches + common_ancestor = repo.merge_base(from_branch, to_branch) + + print(f"Common ancestor is {common_ancestor}") + + # Get the commits in 'from_branch' that are not in 'to_branch' + commits = list(repo.iter_commits(f"{to_branch}..{from_branch}")) + +# for commit in commits: +# print(commit.hexsha, commit.message.strip()) + +def file_prepend(file, str): + with open(file, 'r') as fd: + contents = fd.read() + new_contents = str + contents + + # Overwrite file but now with prepended string on it + with open(file, 'w') as fd: + fd.write(new_contents) + +def process_git_request(fname, target_branch, source_branch, prj_dir): + retcode = 0 # presume success + file = open(fname, "w") + working_dir = prj_dir + os.chdir(working_dir) + + git_cmd = f"git log -1 --format=%H origin/{target_branch}" + gitbr_out, gitbr_err = subprocess.Popen(git_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() + if gitbr_err: + print(f"git log -1 returned error {gitbr_err}") + else: + gitbr_lines = gitbr_out.splitlines() + for x in gitbr_lines: + print(f"git log -1 output line {x}") + local_target_branch = x + + git_cmd = f"git log -1 --format=%H origin/{source_branch}" + gitbr_out, gitbr_err = subprocess.Popen(git_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() + if gitbr_err: + print(f"git log -1 returned error {gitbr_err}") + else: + gitbr_lines = gitbr_out.splitlines() + for x in gitbr_lines: + print(f"git log -1 output line {x}") + local_source_branch = x + +# git_cmd = f"git log --oneline --no-abbrev-commit " + local_target_branch + ".." + local_source_branch + "-- ." +# print(f"git command is {git_cmd}")a +# log_commits_between_branches(prj_dir, source_branch, target_branch) + repo = git.Repo(".") + commits = repo.iter_commits(f"{target_branch}..{source_branch}") + for commit in commits: + print(f"{commit.hexsha[:7]} {commit.message.splitlines()[0]}") + return 0 + loglines_to_check = 13 + try: + out = subprocess.run(git_cmd, shell=True, capture_output=True, text=True, encoding='latin-1') + if out.returncode: + print(f"Command error output is {out}") + file.write(f"Command error output is {out}") + file.close() + return 1 + else: + print(f"Git log line executed") + + line_count = len(str(out.stdout).splitlines()) + print(f"Got {line_count} lines of git log output") + if line_count > 1000: + print(f"Huge Line count {line_count}") + return 0 + + output_lines = out.stdout + print(f"{output_lines}") + return 0 + commit_sha = "" + # we just want the commit sha IDs + for x in output_lines: +# print(f"This is output_lines {x}") + if not bool(re.search(r'[^\x30-\x39a-fA-F]', x)): # equivalent to Ruby's !x[/\H/] + continue + else: + y = x.split() + commit_sha = str(y[0]) + print(f"Found a commit in line ", commit_sha) + + git_cmd = "git show " + commit_sha + gitlog_out, gitlog_err = subprocess.Popen(git_cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True).communicate() + + loglines = gitlog_out.splitlines() + lines_counted = 0 + local_diffdiff_sha = commit_sha + upstream_diffdiff_sha = "" + upstream_diff = False + + for logline in loglines: +# print(f"Processing logline {commit_sha}") + lines_counted += 1 + if lines_counted == 1: + file.write("Merge Request sha: " + local_diffdiff_sha) + file.write("\n") + if lines_counted == 2: # email address + if "ciq.com" not in logline.lower(): + # Bad Author + s = f"error:\nBad {logline}\n" + print(s) + file.write(s) + file.close() + return retcode + if lines_counted > 1: + if "jira" in logline.lower(): + file.write("\t" + logline + "\n") + + if "upstream-diff" in logline.lower(): + upstream_diff = True + + if "commit" in logline.lower(): + commit_sha = re.search(r'[0-9a-f]{40}', logline) + upstream_diffdiff_sha = str(commit_sha.group(0)) if commit_sha else "" + print(f"Upstream : " + upstream_diffdiff_sha) + if upstream_diffdiff_sha: + file.write("\tUpstream sha: " + upstream_diffdiff_sha) + file.write("\n") + + if lines_counted > loglines_to_check: # Everything we need should be in the first loglines_to_check lines + # print(f"Breaking after {loglines_to_check} lines of commit checking") + break + + if local_diffdiff_sha and upstream_diffdiff_sha: + diff_cmd = os.path.join(os.getcwd(), ".github/workflows/diffdiff.py") + " --colour --commit " + local_diffdiff_sha + # print("diffdiff: " + diff_cmd) + process = subprocess.run(diff_cmd, shell=True, capture_output=True, text=True) + diff_out = process.stdout + diff_err = process.stderr + diff_status = process.returncode + + if diff_status != 0 and not upstream_diff: + print(f"diffdiff out: " + diff_out) + print(f"diffdiff err: " + diff_err) + retcode = 1 + file.write("error:\nCommit: " + local_diffdiff_sha + " differs with no upstream tag in commit message\n") + except Exception as error: + print(f"Exception in git log command error {error}") + + finally: + file.close() + + return retcode + +first_arg, *argv_in = sys.argv[1:] # Skip script name in sys.argv + +if len(argv_in) < 5: + print("Not enough arguments: fname, target_branch, source_branch, prj_dir, pull_request, requestor") + sys.exit() + +fname = str(first_arg) +fname = "tmp-" + fname +# print("filename is " + fname) +target_branch = str(argv_in[0]) +# print("target branch is " + target_branch) +source_branch = str(argv_in[1]) +# print("source branch is " + source_branch) +prj_dir = str(argv_in[2]) +# print("project dir is " + prj_dir) +pullreq = str(argv_in[3]) +# print("pull request is " + pullreq) +requestor = str(argv_in[4]) + +retcode = process_git_request(fname, target_branch, pullreq, prj_dir) + +if retcode != 0: + with open(fname, 'r') as fd: + contents = fd.read() + print(contents) + sys.exit(1) +else: + print("Done") + +sys.exit(0) + diff --git a/.github/workflows/process-git-request.rb b/.github/workflows/process-git-request.rb deleted file mode 100644 index 04a2ccd49b8b..000000000000 --- a/.github/workflows/process-git-request.rb +++ /dev/null @@ -1,140 +0,0 @@ -require 'open3' - -requestors = { "gvrose8192" => "" } - -def file_prepend(file, str) - new_contents = "" - File.open(file, 'r') do |fd| - contents = fd.read - new_contents = str << contents - end - # Overwrite file but now with prepended string on it - File.open(file, 'w') do |fd| - fd.write(new_contents) - end -end - -def process_git_request(fname, target_branch, source_branch, prj_dir) - retcode = 200 #presume success -# puts "Opening file " + fname - file = File.new(fname, "w") - working_dir = prj_dir -# puts "Working Dir : " + working_dir - Dir.chdir working_dir -# puts "pwd : " + Dir.pwd - git_cmd = "git log --oneline --no-abbrev-commit origin/" + target_branch + ".." + "origin/" + source_branch -# puts git_cmd - out, err, status = Open3.capture3(git_cmd) - if status.exitstatus != 0 - puts "Command error output is " + err - file.write("Command error output is " + err) - file.close - retcode = 201 - return retcode - end - output_lines = out.split(' ') -# we just want the commit sha IDs - output_lines.each { |x| -# puts "This is output_lines " + x - upstream_diff = false - if !x[/\H/] - if x.length < 40 - next - end - git_cmd = "git show " + x - gitlog_out, gitlog_err, gitlog_status = Open3.capture3(git_cmd) - if gitlog_status.exitstatus != 0 - file.write("git show command error output is " + gitlog_err) - retcode = 201 - end - loglines = gitlog_out.lines.map(&:chomp) - lines_counted = 0 - local_diffdiff_sha = "" - upstream_diffdiff_sha = "" - loglines.each { |logline| - lines_counted = lines_counted + 1 - if lines_counted == 1 - local_commit_sha = logline.match("[0-9a-f]\{40\}") - local_diffdiff_sha = local_commit_sha.to_s -# puts "Local : " + local_diffdiff_sha - file.write("Merge Request sha: " + local_diffdiff_sha) - file.write("\n") - end - if lines_counted == 2 #email address - if !logline.downcase.include? "ciq.com" - # Bad Author - s = "error:\nBad " + logline + "\n" - puts s - file.write(s) - retcode = 201 - else - file.write("\t" + logline + "\n") - end - end - if lines_counted > 1 - if logline.downcase.include? "jira" - file.write("\t" + logline + "\n") - end - if logline.downcase.include? "upstream-diff" - upstream_diff = true - end - if logline.downcase.include? "commit" - commit_sha = logline.match("[0-9a-f]\{40\}") - upstream_diffdiff_sha = commit_sha.to_s -# puts "Upstream : " + upstream_diffdiff_sha - if (!upstream_diffdiff_sha.empty?) - file.write("\tUpstream sha: " + upstream_diffdiff_sha) - file.write("\n") - end - end - end - if lines_counted > 8 #Everything we need should be in the first 8 lines - break - end - } - if !local_diffdiff_sha.empty? && !upstream_diffdiff_sha.empty? - diff_cmd = Dir.pwd + "/.github/workflows/diffdiff.py --colour --commit " + local_diffdiff_sha - puts "diffdiff: " + diff_cmd - diff_out, diff_err, diff_status = Open3.capture3(diff_cmd) - if diff_status.exitstatus != 0 && !upstream_diff - puts "diffdiff out: " + diff_out - puts "diffdiff err: " + diff_err - retcode = 201 - file.write("error:\nCommit: " + local_diffdiff_sha + " differs with no upstream tag in commit message\n") - end - end - end - } - file.close - return retcode -end - -first_arg, *argv_in = ARGV -if argv_in.length < 5 - puts "Not enough arguments: fname, target_branch, source_branch, prj_dir, pull_request, requestor" - exit -end -fname = first_arg.to_s -fname = "tmp-" + fname -# puts "filename is " + fname -target_branch = argv_in[0].to_s -# puts "target branch is " + target_branch -source_branch = argv_in[1].to_s -# puts "source branch is " + source_branch -prj_dir = argv_in[2].to_s -# puts "project dir is " + prj_dir -pullreq = argv_in[3].to_s -# puts "pull request is " + pullreq -requestor = argv_in[4].to_s -retcode = process_git_request(fname, target_branch, source_branch, prj_dir) -if retcode != 200 - File.open(fname, 'r') do |fd| - contents = fd.read - puts contents - end - exit(1) -else - puts "Done" -end -exit(0) - diff --git a/.github/workflows/process-pull-request.yml b/.github/workflows/process-pull-request.yml index 706b4e2d339a..8fc932518bc9 100644 --- a/.github/workflows/process-pull-request.yml +++ b/.github/workflows/process-pull-request.yml @@ -18,33 +18,25 @@ jobs: test: runs-on: ubuntu-latest - strategy: - matrix: - ruby-version: ['3.0'] steps: - uses: actions/checkout@v4 - - name: Set up Ruby - # To automatically get bug fixes and new Ruby versions for ruby/setup-ruby, - # change this to (see https://github.com/ruby/setup-ruby#versioning): - uses: ruby/setup-ruby@v1 - # uses: ruby/setup-ruby@55283cc23133118229fd3f97f9336ee23a179fcf # v1.146.0 - with: - ruby-version: ${{ matrix.ruby-version }} - bundler-cache: true # runs 'bundle install' and caches installed gems automatically - name: Set up Python uses: actions/setup-python@v5 - name: Run tests run: | /usr/bin/pip3 install gitPython python -c "import sys; import git; print(sys.version)" - git fetch origin ${{ github.base_ref }} - if ! git fetch origin ${{ github.head_ref }}; then - echo "Unable to checkout ${{ github.head_ref }}" - fi - git remote add linux https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git - git fetch --shallow-since="3 years ago" linux - echo "Will run process-git-request.rb with:" + rm -rf /home/runner/work/kernel-src-tree/kernel-src-tree + cd /home/runner/work/kernel-src-tree + git clone https://github.com/ctrliq/kernel-src-tree + cd kernel-src-tree + git fetch --all origin + git checkout -b ${{ github.head_ref }} origin/${{ github.head_ref }} + git checkout ${{ github.base_ref }} + # git remote add linux https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git + # git fetch --shallow-since="3 years ago" linux + echo "Will run process-git-request.py with:" echo "fname = ${{ github.run_id }}" echo "target_branch = ${{ github.base_ref }}" echo "source_branch = ${{ github.head_ref }}" @@ -52,5 +44,5 @@ jobs: echo "pull_request = ${{ github.ref }}" echo "requestor = ${{ github.actor }}" cd ${{ github.workspace }} - /usr/bin/ruby .github/workflows/process-git-request.rb ${{ github.run_id }} ${{ github.base_ref }} \ - ${{ github.head_ref }} ${{ github.workspace }} ${{ github.ref }} ${{ github.actor }} + #/usr/bin/python3 .github/workflows/process-git-request.py ${{ github.run_id }} ${{ github.base_ref }} \ + #${{ github.ref }} ${{ github.workspace }} ${{ github.head_ref }} ${{ github.actor }}