Add build machinery

Gemfile

@@ -0,0 +1,22 @@
+source 'https://rubygems.org'
+
+def location_for(place)
+  if place =~ /^(git[:@][^#]*)#(.*)/
+    [{ :git => $1, :branch => $2, :require => false }]
+  elsif place =~ /^file:\/\/(.*)/
+    ['>= 0', { :path => File.expand_path($1), :require => false }]
+  elsif place =~ /(\d+\.\d+\.\d+)/
+    [ place, { :require => false }]
+  end
+end
+
+gem "beaker-hostgenerator", *location_for(ENV['BEAKER_HOSTGENERATOR_VERSION'] || "~> 0.7")
+gem "beaker-abs", *location_for(ENV['BEAKER_ABS_VERSION'] || "~> 0.1")
+# We must do this here in the plumbing branch so that when the build script
+# runs with VANAGON_LOCATION set, it already has the right gem installed.
+# Bundler seems to get confused when the rake tasks runs with a different
+# vanagon version in the bundle.
+gem 'vanagon', *location_for("[email protected]:overlookinfra/vanagon#main")
+gem 'packaging', *location_for(ENV['PACKAGING_LOCATION'] || '~> 0.105')
+gem 'json'
+gem 'rake'

Rakefile

@@ -0,0 +1 @@
+Dir.glob(File.join('tasks/**/*.rake')).each { |file| load file }

build-vanagon.rb

@@ -0,0 +1,144 @@
+#!/usr/bin/env ruby
+
+require 'io/console'
+require 'open3'
+require 'fileutils'
+
+# USAGE: ./build-vanagon.rb <vanagon project name> <folder name> <version> <number of threads>
+
+PLATFORM_LIST = [
+    'debian-11-amd64',
+    'debian-12-amd64',
+    'el-7-x86_64',
+    'el-8-x86_64',
+    'el-9-x86_64',
+    'fedora-40-x86_64',
+    'sles-15-x86_64',
+    'ubuntu-18.04-amd64',
+    'ubuntu-20.04-amd64',
+    'ubuntu-22.04-amd64',
+    'ubuntu-24.04-amd64',
+]
+
+@project = ARGV[0] # Name of project in vanagon
+@repo = ARGV[1] # Name of folder/repo
+@version = ARGV[2]
+# Recommended values for 32 hardware threads and 32GB RAM
+#   puppet-runtime: 8
+#   pxp-agent: 4
+@instance_num = [Integer(ARGV[3]), PLATFORM_LIST.count].min
+
+@thread_messages = {}
+@thread_platforms = {}
+@failed_platforms = []
+@completed_platforms = []
+@mutex = Mutex.new
+@timestamp = Time.now.strftime('%Y%m%d_%H%M%S')
+
+class ThreadPool
+    def initialize(size)
+        @size = size
+        @queue = Queue.new
+        @threads = Array.new(size) do
+            Thread.new do
+                until (task = @queue.pop) == :stop
+                    task.call
+                end
+            end
+        end
+    end
+
+    def schedule(&task)
+        @queue << task
+    end
+
+    def shutdown
+        @size.times { @queue << :stop }
+        @threads.each(&:join)
+    end
+end
+
+def safe_print(thread_id, message)
+    @thread_messages[thread_id] = @thread_messages[thread_id] || []
+    @thread_messages[thread_id] << message
+
+    @mutex.synchronize do
+        print "\e[H\e[2J"
+        print "Building #{@project}\n\n"
+        @thread_messages.keys.sort.each do |i|
+            print "Thread #{i} (#{@thread_platforms[i]}): #{@thread_messages[i][-1]}\n\n"
+        end
+        print "Platforms in the queue: \n#{(PLATFORM_LIST - @thread_platforms.values - @completed_platforms).join("\n")}\n\n"
+        print "Failed platforms: \n#{@failed_platforms.join("\n")}"
+        $stdout.flush
+    end
+end
+
+def thread_message_dump(thread_id)
+    File.open("#{__dir__}/#{@repo}_#{@thread_platforms[thread_id]}_#{@timestamp}.txt", 'w') do |f|
+        f.write(@thread_messages[thread_id].join("\n"))
+    end
+end
+
+def merge_directories(source, target)
+    FileUtils.mkdir_p(target)
+    FileUtils.cp_r(Dir["#{source}/."], target)
+end
+
+def build(thread_id, platform, repo, project)
+    Thread.current.report_on_exception = false
+    safe_print(thread_id, "Starting build of #{platform}")
+    dir = "/tmp/#{repo}.#{thread_id}.#{@timestamp}"
+    FileUtils.cp_r("#{__dir__}/../#{repo}", dir)
+    begin
+        run(thread_id, dir, "git checkout #{@version}")
+        run(thread_id, dir, "rm -rf .bundle")
+        run(thread_id, dir, "rm -f Gemfile.lock")
+        run(thread_id, dir, "bundle install")
+        run(thread_id, dir, "rm -rf output")
+        run(thread_id, dir, "bundle exec build #{project} #{platform} --engine docker")
+        merge_directories("#{dir}/output", "#{__dir__}/../#{repo}/output")
+        safe_print(thread_id, "Done building #{platform}")
+    rescue Exception => e
+        @mutex.synchronize { @failed_platforms << platform }
+        safe_print(thread_id, e)
+        thread_message_dump(thread_id)
+    ensure
+        FileUtils.rm_rf(dir)
+    end
+end
+
+def run(thread_id, dir, command)
+    Open3.popen2e(command, chdir: dir) do |stdin, stdout_stderr, wait_thr|
+        stdout_stderr.each_line do |line|
+            safe_print(thread_id, line.chomp)
+        end
+        exit_status = wait_thr.value
+        unless exit_status.success?
+            raise "Command failed with exit status: #{exit_status.exitstatus}"
+        end
+    end
+end
+
+start = Time.now
+pool = ThreadPool.new(@instance_num)
+
+PLATFORM_LIST.each do |plat|
+    pool.schedule do
+        thread_id = "#{Process.pid}-#{Thread.current.object_id}"
+        @mutex.synchronize do
+            @thread_messages[thread_id] = []
+            @thread_platforms[thread_id] = plat
+        end
+        build(thread_id, plat, @repo, @project)
+        @mutex.synchronize { @completed_platforms << plat }
+    end
+end
+
+pool.shutdown
+
+unless @failed_platforms.empty?
+    puts "Failed platforms:"
+    puts @failed_platforms.join("\n")
+end
+puts "Total time: #{Time.now - start}"

overlookinfra.patch

@@ -0,0 +1,33 @@
+diff --git a/configs/platforms/debian-12-amd64.rb b/configs/platforms/debian-12-amd64.rb
+index bb3e3ab..a5f0cce 100644
+--- a/configs/platforms/debian-12-amd64.rb
++++ b/configs/platforms/debian-12-amd64.rb
+@@ -1,3 +1,5 @@
+ platform "debian-12-amd64" do |plat|
+   plat.inherit_from_default
++  packages = %w(git)
++  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
+ end
+\ No newline at end of file
+diff --git a/configs/platforms/fedora-40-x86_64.rb b/configs/platforms/fedora-40-x86_64.rb
+index 8c6bda6..f5e9353 100644
+--- a/configs/platforms/fedora-40-x86_64.rb
++++ b/configs/platforms/fedora-40-x86_64.rb
+@@ -1,3 +1,5 @@
+ platform 'fedora-40-x86_64' do |plat|
+   plat.inherit_from_default
++  packages = %w(git)
++  plat.provision_with("yum install -y #{packages.join(' ')}")
+ end
+\ No newline at end of file
+diff --git a/configs/platforms/ubuntu-24.04-amd64.rb b/configs/platforms/ubuntu-24.04-amd64.rb
+new file mode 100644
+index 0000000..c1a1719
+--- /dev/null
++++ b/configs/platforms/ubuntu-24.04-amd64.rb
+@@ -0,0 +1,5 @@
++platform "ubuntu-24.04-amd64" do |plat|
++  plat.inherit_from_default
++  packages = %w(git)
++  plat.provision_with "export DEBIAN_FRONTEND=noninteractive; apt-get update -qq; apt-get install -qy --no-install-recommends #{packages.join(' ')}"
++end

tasks/build.rake

@@ -0,0 +1,27 @@
+require 'open3'
+
+# Tested on a system with 32 threads and 32GB of RAM. Adjust threads accordingly.
+namespace :overlookinfra do
+  desc "Build a given project at a given tag. This should be a tag created via the overlookinfra:tag task."
+  task :build, [:project, :tag, :threads] do |t, args|
+    args.with_defaults(threads: 8)
+    threads = Integer(args[:threads])
+
+    args.with_defaults(project: 'puppet-bolt')
+    project = args[:project]
+
+    vanagon = 'VANAGON_LOCATION="[email protected]:overlookinfra/vanagon#main"'
+    if args[:tag].nil? || args[:tag].empty?
+      abort "You must provide a tag."
+      return
+    end
+
+    cmd = "#{vanagon} #{__dir__}/../build-vanagon.rb #{project} bolt-vanagon #{args[:tag]} #{threads}"
+    puts "Running #{cmd}"
+    Open3.popen2e(cmd) do |stdin, stdout_stderr, thread|
+      stdout_stderr.each { |line| puts line }
+      exit_status = thread.value
+      puts "Command finished with status #{exit_status.exitstatus}"
+    end
+  end
+end

tasks/tag.rake

@@ -0,0 +1,54 @@
+require 'open3'
+namespace :overlookinfra do
+  desc "Apply overlookinfra changes to given tag, and create new tag. First argument is the puppetlabs tag, second is the overlook tag for puppet-runtime."
+  task :tag, [:tag, :puppet_runtime_tag] do |t, args|
+    patch_branch = 'plumbing'
+    patch_file = 'overlookinfra.patch'
+    if args[:tag].nil? || args[:tag].empty?
+      abort "You must provide a tag."
+    end
+    if args[:puppet_runtime_tag].nil? || args[:puppet_runtime_tag].empty?
+      abort "You must provide a tag for puppet-runtime that has been uploaded to s3.osuosl.org."
+    end
+    branch = "overlookinfra/#{args[:tag]}"
+    tag = "#{args[:tag]}-overlookinfra"
+
+    puts "Checking out #{args[:tag]}"
+    run_command("git checkout #{args[:tag]}")
+
+    puts "Checking out #{patch_file}"
+    run_command("git checkout #{patch_branch} -- #{patch_file}")
+
+    puts "Applying patch"
+    run_command("git apply #{patch_file}")
+
+    puts "Replacing puppet-runtime.json"
+    munged = args[:puppet_runtime_tag].gsub('-','.')
+    data = <<~DATA
+      {"location":"https://s3.osuosl.org/puppet-artifacts/puppet-runtime/#{args[:puppet_runtime_tag]}/","version":"#{munged}"}
+    DATA
+    File.write("configs/components/puppet-runtime.json",data)
+
+    puts "Creating commit"
+    run_command("git add .")
+    run_command("git rm -f #{patch_file}")
+    run_command("git commit -m \"#{tag}\"")
+
+    puts "Creating tag #{tag}"
+    sha = run_command("git rev-parse HEAD")
+    run_command("git tag -a #{tag} -m #{tag} #{sha}")
+
+    puts "Creating branch #{branch}"
+    run_command("git checkout -b #{branch}")
+
+    puts "Pushing to origin"
+    run_command("git push origin #{branch}")
+    run_command("git push origin #{tag}")
+  end
+end
+
+def run_command(cmd)
+  output, status = Open3.capture2e(cmd)
+  abort "Command failed! Command: #{cmd}, Output: #{output}" unless status.exitstatus.zero?
+  return output.chomp
+end

tasks/upload.rake

@@ -0,0 +1,40 @@
+require 'open3'
+
+namespace :overlookinfra do
+  desc "Upload artifacts from the output directory to S3. Requires the AWS CLI to be installed and configured appropriately."
+  task :upload, [:tag] do |t, args|
+    endpoint = ENV['ENDPOINT_URL']
+    bucket = ENV['BUCKET_NAME']
+
+    if endpoint.nil? || endpoint.empty?
+      abort "You must set the ENDPOINT_URL environment variable to the S3 server you want to upload to."
+    end
+    if bucket.nil? || bucket.empty?
+      abort "You must set the BUCKET_NAME environment variable to the S3 bucket you are uploading to."
+    end
+    if args[:tag].nil? || args[:tag].empty?
+      abort "You must provide a tag."
+    end
+    munged_tag = args[:tag].gsub('-','.')
+    s3 = "aws s3 --endpoint-url=#{endpoint}"
+
+    # Ensure the AWS CLI isn't going to fail with the given parameters
+    run_command("#{s3} ls s3://#{bucket}/")
+
+    files = Dir.glob("#{__dir__}/../output/**/*#{munged_tag}*")
+    if files.empty?
+      puts "No files for the given tag found in the output directory."
+    end
+    path = "s3://#{bucket}/bolt/#{args[:tag]}"
+    files.each do |f|
+      puts "Uploading #{File.basename(f)}"
+      run_command("#{s3} cp #{f} #{path}/#{File.basename(f)} --endpoint-url=#{endpoint}")
+    end
+  end
+end
+
+def run_command(cmd)
+  output, status = Open3.capture2e(cmd)
+  abort "Command failed! Command: #{cmd}, Output: #{output}" unless status.exitstatus.zero?
+  return output.chomp
+end