From 6f99704a649802c1b62acac9b38e5659a67bd0b0 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Fri, 7 Feb 2025 20:05:05 +0000 Subject: [PATCH] Import `brew formula-analytics` and `generate-analytics-api` commands Import these from the homebrew/formula-analytics tap and deprecate that tap. This required a little messing around with filenames and paths to get it finding Python and writing to the user's home directory. --- .github/dependabot.yml | 7 + .github/workflows/tests.yml | 48 ++- .gitignore | 1 + Library/Homebrew/Gemfile | 3 + Library/Homebrew/Gemfile.lock | 2 + Library/Homebrew/dev-cmd/formula-analytics.rb | 392 ++++++++++++++++++ .../dev-cmd/generate-analytics-api.rb | 138 ++++++ .../formula-analytics/.python-version | 1 + .../formula-analytics/pycall-setup.rb | 12 + .../formula-analytics/pycall-setup.rbi | 24 ++ .../formula-analytics/requirements.in | 1 + .../formula-analytics/requirements.txt | 84 ++++ .../transform_analytics_to_counts.json | 15 + .../homebrew/dev_cmd/formula_analytics.rbi | 61 +++ .../dev_cmd/generate_analytics_api.rbi | 13 + Library/Homebrew/sorbet/tapioca/config.yml | 3 + .../test/dev-cmd/formula-analytics_spec.rb | 8 + .../dev-cmd/generate-analytics-api_spec.rb | 8 + 18 files changed, 816 insertions(+), 5 deletions(-) create mode 100755 Library/Homebrew/dev-cmd/formula-analytics.rb create mode 100755 Library/Homebrew/dev-cmd/generate-analytics-api.rb create mode 100644 Library/Homebrew/formula-analytics/.python-version create mode 100644 Library/Homebrew/formula-analytics/pycall-setup.rb create mode 100644 Library/Homebrew/formula-analytics/pycall-setup.rbi create mode 100644 Library/Homebrew/formula-analytics/requirements.in create mode 100644 Library/Homebrew/formula-analytics/requirements.txt create mode 100644 Library/Homebrew/formula-analytics/transform_analytics_to_counts.json create mode 100644 Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/formula_analytics.rbi create mode 100644 Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/generate_analytics_api.rbi create mode 100644 Library/Homebrew/test/dev-cmd/formula-analytics_spec.rb create mode 100644 Library/Homebrew/test/dev-cmd/generate-analytics-api_spec.rb diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 65452dedcd1f9..417a3aaeac4b9 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -53,3 +53,10 @@ updates: interval: daily allow: - dependency-type: all + + - package-ecosystem: pip + directory: /Library/Homebrew/formula-analytics/ + schedule: + interval: daily + allow: + - dependency-type: all diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bf9a7e49b309a..d165b0fec6787 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -115,7 +115,6 @@ jobs: run: | brew tap homebrew/bundle brew tap homebrew/command-not-found - brew tap homebrew/formula-analytics brew tap homebrew/portable-ruby brew tap homebrew/services @@ -129,7 +128,6 @@ jobs: homebrew/test-bot brew style homebrew/command-not-found \ - homebrew/formula-analytics \ homebrew/portable-ruby - name: Run brew style on homebrew/cask @@ -182,9 +180,6 @@ jobs: run: | brew audit --skip-style --except=version --tap=homebrew/cask - - name: Generate formula API - run: brew generate-formula-api --dry-run - - name: Generate cask API run: brew generate-cask-api --dry-run @@ -398,3 +393,46 @@ jobs: - run: brew install gnu-tar - run: brew test-bot --only-formulae --only-json-tab --test-default-formula + + test-analytics: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - macos-latest + needs: syntax + if: github.repository_owner == 'Homebrew' && github.event_name != 'push' + steps: + - name: Set up Homebrew + id: set-up-homebrew + uses: Homebrew/actions/setup-homebrew@master + + - name: Setup Python + uses: actions/setup-python@42375524e23c412d93fb67b49958b491fce71c38 # v5.4.0 + with: + python-version-file: ${{ steps.set-up-homebrew.outputs.repository-path }}/Library/Homebrew/formula-analytics/.python-version + + - name: Cache Homebrew Bundler RubyGems + id: cache + uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0 + with: + path: ${{ steps.set-up-homebrew.outputs.gems-path }} + key: ${{ runner.os }}-rubygems-${{ steps.set-up-homebrew.outputs.gems-hash }} + restore-keys: ${{ runner.os }}-rubygems- + + - name: Install Homebrew Bundler RubyGems + if: steps.cache.outputs.cache-hit != 'true' + run: brew install-bundler-gems + + - run: brew formula-analytics --setup + + - run: brew formula-analytics --install --json --days-ago=2 + if: github.event.pull_request.head.repo.fork == false && (github.event_name == 'pull_request' && github.event.pull_request.user.login != 'dependabot[bot]') + env: + HOMEBREW_INFLUXDB_TOKEN: ${{ secrets.HOMEBREW_INFLUXDB_READ_TOKEN }} + + - run: brew generate-analytics-api + if: github.event.pull_request.head.repo.fork == false && (github.event_name == 'pull_request' && github.event.pull_request.user.login != 'dependabot[bot]') + env: + HOMEBREW_INFLUXDB_TOKEN: ${{ secrets.HOMEBREW_INFLUXDB_READ_TOKEN }} diff --git a/.gitignore b/.gitignore index 5e9ff351e9e66..8823009012c8d 100644 --- a/.gitignore +++ b/.gitignore @@ -106,6 +106,7 @@ **/vendor/bundle/ruby/*/gems/prism-*/ **/vendor/bundle/ruby/*/gems/psych-*/ **/vendor/bundle/ruby/*/gems/pry-*/ +**/vendor/bundle/ruby/*/gems/pycall-*/ **/vendor/bundle/ruby/*/gems/racc-*/ **/vendor/bundle/ruby/*/gems/rainbow-*/ **/vendor/bundle/ruby/*/gems/rbi-*/ diff --git a/Library/Homebrew/Gemfile b/Library/Homebrew/Gemfile index 317ce59571ee6..764f615c4331a 100644 --- a/Library/Homebrew/Gemfile +++ b/Library/Homebrew/Gemfile @@ -71,6 +71,9 @@ end group :vscode, optional: true do gem "ruby-lsp", require: false end +group :formula_analytics, optional: true do + gem "pycall", require: false +end # shared gems (used by multiple groups) group :audit, :bump_unversioned_casks, :livecheck, optional: true do diff --git a/Library/Homebrew/Gemfile.lock b/Library/Homebrew/Gemfile.lock index a2ba8beff563c..e52731a706058 100644 --- a/Library/Homebrew/Gemfile.lock +++ b/Library/Homebrew/Gemfile.lock @@ -43,6 +43,7 @@ GEM coderay (~> 1.1) method_source (~> 1.0) public_suffix (6.0.1) + pycall (1.5.2) racc (1.8.1) rainbow (3.1.1) rbi (0.2.4) @@ -168,6 +169,7 @@ DEPENDENCIES patchelf plist pry + pycall redcarpet rexml rspec diff --git a/Library/Homebrew/dev-cmd/formula-analytics.rb b/Library/Homebrew/dev-cmd/formula-analytics.rb new file mode 100755 index 0000000000000..9d92285a5738c --- /dev/null +++ b/Library/Homebrew/dev-cmd/formula-analytics.rb @@ -0,0 +1,392 @@ +# typed: strict +# frozen_string_literal: true + +require "abstract_command" + +module Homebrew + module DevCmd + class FormulaAnalytics < AbstractCommand + cmd_args do + usage_banner <<~EOS + `formula-analytics` + + Query Homebrew's analytics. + EOS + flag "--days-ago=", + description: "Query from the specified days ago until the present. The default is 30 days." + switch "--install", + description: "Output the number of specifically requested installations or installation as " \ + "dependencies of the formula. This is the default." + switch "--cask-install", + description: "Output the number of installations of casks." + switch "--install-on-request", + description: "Output the number of specifically requested installations of the formula." + switch "--build-error", + description: "Output the number of build errors for the formulae." + switch "--os-version", + description: "Output OS versions." + switch "--homebrew-devcmdrun-developer", + description: "Output devcmdrun/HOMEBREW_DEVELOPER." + switch "--homebrew-os-arch-ci", + description: "Output OS/Architecture/CI." + switch "--homebrew-prefixes", + description: "Output Homebrew prefixes." + switch "--homebrew-versions", + description: "Output Homebrew versions." + switch "--brew-command-run", + description: "Output `brew` commands run." + switch "--brew-command-run-options", + description: "Output `brew` commands run with options." + switch "--brew-test-bot-test", + description: "Output `brew test-bot` steps run." + switch "--json", + description: "Output JSON. This is required: plain text support has been removed." + switch "--all-core-formulae-json", + description: "Output a different JSON format containing the JSON data for all " \ + "Homebrew/homebrew-core formulae." + switch "--setup", + description: "Install the necessary gems, require them and exit without running a query." + conflicts "--install", "--cask-install", "--install-on-request", "--build-error", "--os-version", + "--homebrew-devcmdrun-developer", "--homebrew-os-arch-ci", "--homebrew-prefixes", + "--homebrew-versions", "--brew-command-run", "--brew-command-run-options", "--brew-test-bot-test" + conflicts "--json", "--all-core-formulae-json", "--setup" + named_args :none + end + + FIRST_INFLUXDB_ANALYTICS_DATE = T.let(Date.new(2023, 03, 27).freeze, Date) + + sig { override.void } + def run + Homebrew.install_bundler_gems!(groups: ["formula_analytics"]) + + setup_python + influx_analytics(args) + end + + sig { void } + def setup_python + formula_analytics_root = HOMEBREW_LIBRARY/"Homebrew/formula-analytics" + vendor_python = Pathname.new("~/.brew-formula-analytics/vendor/python").expand_path + python_version = (formula_analytics_root/".python-version").read.chomp + + which_python = which("python#{python_version}", ORIGINAL_PATHS) + odie <<~EOS if which_python.nil? + Python #{python_version} is required. Try: + brew install python@#{python_version} + EOS + + venv_root = vendor_python/python_version + vendor_python.children.reject { |path| path == venv_root }.each(&:rmtree) if vendor_python.exist? + venv_python = venv_root/"bin/python" + + repo_requirements = HOMEBREW_LIBRARY/"Homebrew/formula-analytics/requirements.txt" + venv_requirements = venv_root/"requirements.txt" + if !venv_requirements.exist? || !FileUtils.identical?(repo_requirements, venv_requirements) + safe_system which_python, "-I", "-m", "venv", "--clear", venv_root, out: :err + safe_system venv_python, "-m", "pip", "install", + "--disable-pip-version-check", + "--require-hashes", + "--requirement", repo_requirements, + out: :err + FileUtils.cp repo_requirements, venv_requirements + end + + ENV["PATH"] = "#{venv_root}/bin:#{ENV.fetch("PATH")}" + ENV["__PYVENV_LAUNCHER__"] = venv_python.to_s # support macOS framework Pythons + + require "pycall" + PyCall.init(venv_python) + require formula_analytics_root/"pycall-setup" + end + + sig { params(args: Homebrew::DevCmd::FormulaAnalytics::Args).void } + def influx_analytics(args) + require "utils/analytics" + require "json" + + return if args.setup? + + odie "HOMEBREW_NO_ANALYTICS is set!" if ENV["HOMEBREW_NO_ANALYTICS"] + + token = ENV.fetch("HOMEBREW_INFLUXDB_TOKEN", nil) + odie "No InfluxDB credentials found in HOMEBREW_INFLUXDB_TOKEN!" unless token + + client = InfluxDBClient3.new( + token:, + host: URI.parse(Utils::Analytics::INFLUX_HOST).host, + org: Utils::Analytics::INFLUX_ORG, + database: Utils::Analytics::INFLUX_BUCKET, + ) + + max_days_ago = (Date.today - FIRST_INFLUXDB_ANALYTICS_DATE).to_s.to_i + days_ago = (args.days_ago || 30).to_i + if days_ago > max_days_ago + opoo "Analytics started #{FIRST_INFLUXDB_ANALYTICS_DATE}. `--days-ago` set to maximum value." + days_ago = max_days_ago + end + if days_ago > 365 + opoo "Analytics are only retained for 1 year, setting `--days-ago=365`." + days_ago = 365 + end + + all_core_formulae_json = args.all_core_formulae_json? + + categories = [] + categories << :build_error if args.build_error? + categories << :cask_install if args.cask_install? + categories << :formula_install if args.install? + categories << :formula_install_on_request if args.install_on_request? + categories << :homebrew_devcmdrun_developer if args.homebrew_devcmdrun_developer? + categories << :homebrew_os_arch_ci if args.homebrew_os_arch_ci? + categories << :homebrew_prefixes if args.homebrew_prefixes? + categories << :homebrew_versions if args.homebrew_versions? + categories << :os_versions if args.os_version? + categories << :command_run if args.brew_command_run? + categories << :command_run_options if args.brew_command_run_options? + categories << :test_bot_test if args.brew_test_bot_test? + + category_matching_buckets = [:build_error, :cask_install, :command_run, :test_bot_test] + + categories.each do |category| + additional_where = all_core_formulae_json ? " AND tap_name ~ '^homebrew/(core|cask)$'" : "" + bucket = if category_matching_buckets.include?(category) + category + elsif category == :command_run_options + :command_run + else + :formula_install + end + + case category + when :homebrew_devcmdrun_developer + dimension_key = "devcmdrun_developer" + groups = [:devcmdrun, :developer] + when :homebrew_os_arch_ci + dimension_key = "os_arch_ci" + groups = [:os, :arch, :ci] + when :homebrew_prefixes + dimension_key = "prefix" + groups = [:prefix, :os, :arch] + when :homebrew_versions + dimension_key = "version" + groups = [:version] + when :os_versions + dimension_key = :os_version + groups = [:os_name_and_version] + when :command_run + dimension_key = "command_run" + groups = [:command] + when :command_run_options + dimension_key = "command_run_options" + groups = [:command, :options, :devcmdrun, :developer] + additional_where += " AND ci = 'false'" + when :test_bot_test + dimension_key = "test_bot_test" + groups = [:command, :passed, :arch, :os] + when :cask_install + dimension_key = :cask + groups = [:package, :tap_name] + else + dimension_key = :formula + additional_where += " AND on_request = 'true'" if category == :formula_install_on_request + groups = [:package, :tap_name, :options] + end + + sql_groups = groups.map { |e| "\"#{e}\"" }.join(",") + query = <<~EOS + SELECT #{sql_groups}, COUNT(*) AS "count" FROM "#{bucket}" WHERE time >= now() - INTERVAL '#{days_ago} day'#{additional_where} GROUP BY #{sql_groups} + EOS + batches = begin + client.query(query:, language: "sql").to_batches + rescue PyCall::PyError => e + if e.message.include?("message: unauthenticated") + odie "Could not authenticate with InfluxDB! Please check your HOMEBREW_INFLUXDB_TOKEN!" + end + raise + end + + json = T.let({ + category:, + total_items: 0, + start_date: Date.today - days_ago.to_i, + end_date: Date.today, + total_count: 0, + items: [], + }, T::Hash[Symbol, T.untyped]) + + batches.each do |batch| + batch.to_pylist.each do |record| + dimension = case category + when :homebrew_devcmdrun_developer + "devcmdrun=#{record["devcmdrun"]} HOMEBREW_DEVELOPER=#{record["developer"]}" + when :homebrew_os_arch_ci + if record["ci"] == "true" + "#{record["os"]} #{record["arch"]} (CI)" + else + "#{record["os"]} #{record["arch"]}" + end + when :homebrew_prefixes + if record["prefix"] == "custom-prefix" + "#{record["prefix"]} (#{record["os"]} #{record["arch"]})" + else + (record["prefix"]).to_s + end + when :os_versions + format_os_version_dimension(record["os_name_and_version"]) + when :command_run_options + options = record["options"].split + + # Cleanup bad data before TODO + # Can delete this code after 18th July 2025. + options.reject! { |option| option.match?(/^--with(out)?-/) } + next if options.any? { |option| option.match?(/^TMPDIR=/) } + + "#{record["command"]} #{options.sort.join(" ")}" + when :test_bot_test + command_and_package, options = record["command"].split.partition { |arg| !arg.start_with?("-") } + + # Cleanup bad data before https://github.com/Homebrew/homebrew-test-bot/pull/1043 + # Can delete this code after 27th April 2025. + next if %w[audit install linkage style test].exclude?(command_and_package.first) + next if command_and_package.last.include?("/") + next if options.include?("--tap=") + next if options.include?("--only-dependencies") + next if options.include?("--cached") + + command_and_options = (command_and_package + options.sort).join(" ") + passed = (record["passed"] == "true") ? "PASSED" : "FAILED" + + "#{command_and_options} (#{record["os"]} #{record["arch"]}) (#{passed})" + else + record[groups.first.to_s] + end + next if dimension.blank? + + if (tap_name = record["tap_name"].presence) && + ((tap_name != "homebrew/cask" && dimension_key == :cask) || + (tap_name != "homebrew/core" && dimension_key == :formula)) + dimension = "#{tap_name}/#{dimension}" + end + + if (all_core_formulae_json || category == :build_error) && + (options = record["options"].presence) + # homebrew/core formulae don't have non-HEAD options but they ended up in our analytics anyway. + if all_core_formulae_json + options = options.split.include?("--HEAD") ? "--HEAD" : "" + end + dimension = "#{dimension} #{options}" + end + + dimension = dimension.strip + next if dimension.match?(/[<>]/) + + count = record["count"] + + json[:total_items] += 1 + json[:total_count] += count + + json[:items] << { + number: nil, + dimension_key => dimension, + count:, + } + end + end + + odie "No data returned" if json[:total_count].zero? + + # Combine identical values + deduped_items = {} + + json[:items].each do |item| + key = item[dimension_key] + if deduped_items.key?(key) + deduped_items[key][:count] += item[:count] + else + deduped_items[key] = item + end + end + + json[:items] = deduped_items.values + + if all_core_formulae_json + core_formula_items = {} + + json[:items].each do |item| + item.delete(:number) + formula_name, = item[dimension_key].split.first + next if formula_name.include?("/") + + core_formula_items[formula_name] ||= [] + core_formula_items[formula_name] << item + end + json.delete(:items) + + core_formula_items.each_value do |items| + items.sort_by! { |item| -item[:count] } + items.each do |item| + item[:count] = format_count(item[:count]) + end + end + + json[:formulae] = core_formula_items.sort_by { |name, _| name }.to_h + else + json[:items].sort_by! do |item| + -item[:count] + end + + json[:items].each_with_index do |item, index| + item[:number] = index + 1 + + percent = (item[:count].to_f / json[:total_count]) * 100 + item[:percent] = format_percent(percent) + item[:count] = format_count(item[:count]) + end + end + + puts JSON.pretty_generate json + end + end + + sig { params(count: Integer).returns(String) } + def format_count(count) + count.to_s.reverse.gsub(/(\d{3})(?=\d)/, '\\1,').reverse + end + + sig { params(percent: Float).returns(String) } + def format_percent(percent) + format("%.2f", percent:).gsub(/\.00$/, "") + end + + sig { params(dimension: T.nilable(String)).returns(T.nilable(String)) } + def format_os_version_dimension(dimension) + return if dimension.blank? + + dimension = dimension.gsub(/^Intel ?/, "") + .gsub(/^macOS ?/, "") + .gsub(/ \(.+\)$/, "") + case dimension + when "10.11", /^10\.11\.?/ then "OS X El Capitan (10.11)" + when "10.12", /^10\.12\.?/ then "macOS Sierra (10.12)" + when "10.13", /^10\.13\.?/ then "macOS High Sierra (10.13)" + when "10.14", /^10\.14\.?/ then "macOS Mojave (10.14)" + when "10.15", /^10\.15\.?/ then "macOS Catalina (10.15)" + when "10.16", /^11\.?/ then "macOS Big Sur (11)" + when /^12\.?/ then "macOS Monterey (12)" + when /^13\.?/ then "macOS Ventura (13)" + when /^14\.?/ then "macOS Sonoma (14)" + when /^15\.?/ then "macOS Sequoia (15)" + when /Ubuntu(-Server)? (14|16|18|20|22)\.04/ then "Ubuntu #{Regexp.last_match(2)}.04 LTS" + when /Ubuntu(-Server)? (\d+\.\d+).\d ?(LTS)?/ + "Ubuntu #{Regexp.last_match(2)} #{Regexp.last_match(3)}".strip + when %r{Debian GNU/Linux (\d+)\.\d+} then "Debian #{Regexp.last_match(1)} #{Regexp.last_match(2)}" + when /CentOS (\w+) (\d+)/ then "CentOS #{Regexp.last_match(1)} #{Regexp.last_match(2)}" + when /Fedora Linux (\d+)[.\d]*/ then "Fedora Linux #{Regexp.last_match(1)}" + when /KDE neon .*?([\d.]+)/ then "KDE neon #{Regexp.last_match(1)}" + when /Amazon Linux (\d+)\.[.\d]*/ then "Amazon Linux #{Regexp.last_match(1)}" + else dimension + end + end + end + end +end diff --git a/Library/Homebrew/dev-cmd/generate-analytics-api.rb b/Library/Homebrew/dev-cmd/generate-analytics-api.rb new file mode 100755 index 0000000000000..b27a581acc81e --- /dev/null +++ b/Library/Homebrew/dev-cmd/generate-analytics-api.rb @@ -0,0 +1,138 @@ +# typed: strict +# frozen_string_literal: true + +require "abstract_command" + +module Homebrew + module DevCmd + class GenerateAnalyticsApi < AbstractCommand + CATEGORIES = %w[ + build-error install install-on-request + core-build-error core-install core-install-on-request + cask-install core-cask-install os-version + homebrew-devcmdrun-developer homebrew-os-arch-ci + homebrew-prefixes homebrew-versions + brew-command-run brew-command-run-options brew-test-bot-test + ].freeze + + # TODO: add brew-command-run-options brew-test-bot-test to above when working. + DAYS = %w[30 90 365].freeze + MAX_RETRIES = 3 + + cmd_args do + description <<~EOS + Generates analytics API data files for formulae.brew.sh. + + The generated files are written to the current directory. + EOS + + named_args :none + end + + sig { params(category_name: String, data_source: T.nilable(String)).returns(String) } + def analytics_json_template(category_name, data_source: nil) + data_source = "#{data_source}: true" if data_source + + <<~EOS + --- + layout: analytics_json + category: #{category_name} + #{data_source} + --- + {{ content }} + EOS + end + + sig { params(args: String).returns(String) } + def run_formula_analytics(*args) + puts "brew formula-analytics #{args.join(" ")}" + + retries = 0 + result = Utils.popen_read(HOMEBREW_BREW_FILE, "formula-analytics", *args, err: :err) + + while !$CHILD_STATUS.success? && retries < MAX_RETRIES + # Give InfluxDB some more breathing room. + sleep 4**(retries+2) + + retries += 1 + puts "Retrying #{args.join(" ")} (#{retries}/#{MAX_RETRIES})..." + result = Utils.popen_read(HOMEBREW_BREW_FILE, "formula-analytics", *args, err: :err) + end + + odie "`brew formula-analytics #{args.join(" ")}` failed: #{result}" unless $CHILD_STATUS.success? + + result + end + + sig { override.void } + def run + safe_system HOMEBREW_BREW_FILE, "formula-analytics", "--setup" + + directories = ["_data/analytics", "api/analytics"] + FileUtils.rm_rf directories + FileUtils.mkdir_p directories + + root_dir = Pathname.pwd + analytics_data_dir = root_dir/"_data/analytics" + analytics_api_dir = root_dir/"api/analytics" + + threads = [] + + CATEGORIES.each do |category| + formula_analytics_args = [] + + case category + when "core-build-error" + formula_analytics_args << "--all-core-formulae-json" + formula_analytics_args << "--build-error" + category_name = "build-error" + data_source = "homebrew-core" + when "core-install" + formula_analytics_args << "--all-core-formulae-json" + formula_analytics_args << "--install" + category_name = "install" + data_source = "homebrew-core" + when "core-install-on-request" + formula_analytics_args << "--all-core-formulae-json" + formula_analytics_args << "--install-on-request" + category_name = "install-on-request" + data_source = "homebrew-core" + when "core-cask-install" + formula_analytics_args << "--all-core-formulae-json" + formula_analytics_args << "--cask-install" + category_name = "cask-install" + data_source = "homebrew-cask" + else + formula_analytics_args << "--#{category}" + category_name = category + end + + path_suffix = File.join(category_name, data_source || "") + analytics_data_path = analytics_data_dir/path_suffix + analytics_api_path = analytics_api_dir/path_suffix + + FileUtils.mkdir_p analytics_data_path + FileUtils.mkdir_p analytics_api_path + + # The `--json` and `--all-core-formulae-json` flags are mutually + # exclusive, but we need to explicitly set `--json` sometimes, + # so only set it if we've not already set + # `--all-core-formulae-json`. + formula_analytics_args << "--json" unless formula_analytics_args.include? "--all-core-formulae-json" + + DAYS.each do |days| + next if days != "30" && category_name == "build-error" && !data_source.nil? + + threads << Thread.new do + args = %W[--days-ago=#{days}] + (analytics_data_path/"#{days}d.json").write run_formula_analytics(*formula_analytics_args, *args) + (analytics_api_path/"#{days}d.json").write analytics_json_template(category_name, data_source:) + end + end + end + + threads.each(&:join) + end + end + end +end diff --git a/Library/Homebrew/formula-analytics/.python-version b/Library/Homebrew/formula-analytics/.python-version new file mode 100644 index 0000000000000..e4fba21835872 --- /dev/null +++ b/Library/Homebrew/formula-analytics/.python-version @@ -0,0 +1 @@ +3.12 diff --git a/Library/Homebrew/formula-analytics/pycall-setup.rb b/Library/Homebrew/formula-analytics/pycall-setup.rb new file mode 100644 index 0000000000000..70a560ce356c3 --- /dev/null +++ b/Library/Homebrew/formula-analytics/pycall-setup.rb @@ -0,0 +1,12 @@ +# typed: strict +# frozen_string_literal: true + +require "pycall/import" +# This was a rewrite from `include(Module.new(...))`, +# to appease Sorbet, so let's keep the existing behaviour +# and silence RuboCop. +# rubocop:disable Style/MixinUsage +include PyCall::Import +# rubocop:enable Style/MixinUsage + +pyfrom "influxdb_client_3", import: :InfluxDBClient3 diff --git a/Library/Homebrew/formula-analytics/pycall-setup.rbi b/Library/Homebrew/formula-analytics/pycall-setup.rbi new file mode 100644 index 0000000000000..a378238ba626a --- /dev/null +++ b/Library/Homebrew/formula-analytics/pycall-setup.rbi @@ -0,0 +1,24 @@ +# typed: strict + +class InfluxDBClient3 + def self.initialize(*args); end + + def query(*args); end +end + +module PyCall + def self.init(*args); end + + module Import + def self.pyfrom(*args); end + + def self.import(*args); end + end + + PyError = Class.new(StandardError).freeze +end + +# Needs defined here for Sorbet to work as expected. +# rubocop:disable Style/TopLevelMethodDefinition +def pyfrom(*args); end +# rubocop:enable Style/TopLevelMethodDefinition diff --git a/Library/Homebrew/formula-analytics/requirements.in b/Library/Homebrew/formula-analytics/requirements.in new file mode 100644 index 0000000000000..d1ef841565703 --- /dev/null +++ b/Library/Homebrew/formula-analytics/requirements.in @@ -0,0 +1 @@ +influxdb3-python diff --git a/Library/Homebrew/formula-analytics/requirements.txt b/Library/Homebrew/formula-analytics/requirements.txt new file mode 100644 index 0000000000000..a4a2880d799c5 --- /dev/null +++ b/Library/Homebrew/formula-analytics/requirements.txt @@ -0,0 +1,84 @@ +# +# This file is autogenerated by pip-compile with Python 3.12 +# by the following command: +# +# pip-compile --allow-unsafe --generate-hashes --strip-extras requirements.in +# +certifi==2025.1.31 \ + --hash=sha256:3d5da6925056f6f18f119200434a4780a94263f10d1c21d032a6f6b2baa20651 \ + --hash=sha256:ca78db4565a652026a4db2bcdf68f2fb589ea80d0be70e03929ed730746b84fe + # via influxdb3-python +influxdb3-python==0.10.0 \ + --hash=sha256:d279e5f8a597d49b44035263b1cf1472a3861ceba930fd08e1e3b1721a07d3cf \ + --hash=sha256:f3d44dff4c4bbfdcb1fa1c4013ccfa317fbbd7df5812eb46395421166ffb385a + # via -r requirements.in +pyarrow==19.0.0 \ + --hash=sha256:239ca66d9a05844bdf5af128861af525e14df3c9591bcc05bac25918e650d3a2 \ + --hash=sha256:2795064647add0f16563e57e3d294dbfc067b723f0fd82ecd80af56dad15f503 \ + --hash=sha256:29cd86c8001a94f768f79440bf83fee23963af5e7bc68ce3a7e5f120e17edf89 \ + --hash=sha256:2a0144a712d990d60f7f42b7a31f0acaccf4c1e43e957f7b1ad58150d6f639c1 \ + --hash=sha256:2a1a109dfda558eb011e5f6385837daffd920d54ca00669f7a11132d0b1e6042 \ + --hash=sha256:2b6d3ce4288793350dc2d08d1e184fd70631ea22a4ff9ea5c4ff182130249d9b \ + --hash=sha256:2f672f5364b2d7829ef7c94be199bb88bf5661dd485e21d2d37de12ccb78a136 \ + --hash=sha256:3c1c162c4660e0978411a4761f91113dde8da3433683efa473501254563dcbe8 \ + --hash=sha256:450a7d27e840e4d9a384b5c77199d489b401529e75a3b7a3799d4cd7957f2f9c \ + --hash=sha256:4624c89d6f777c580e8732c27bb8e77fd1433b89707f17c04af7635dd9638351 \ + --hash=sha256:4d8b0c0de0a73df1f1bf439af1b60f273d719d70648e898bc077547649bb8352 \ + --hash=sha256:5418d4d0fab3a0ed497bad21d17a7973aad336d66ad4932a3f5f7480d4ca0c04 \ + --hash=sha256:597360ffc71fc8cceea1aec1fb60cb510571a744fffc87db33d551d5de919bec \ + --hash=sha256:5e8a28b918e2e878c918f6d89137386c06fe577cd08d73a6be8dafb317dc2d73 \ + --hash=sha256:62ef8360ff256e960f57ce0299090fb86423afed5e46f18f1225f960e05aae3d \ + --hash=sha256:66732e39eaa2247996a6b04c8aa33e3503d351831424cdf8d2e9a0582ac54b34 \ + --hash=sha256:718947fb6d82409013a74b176bf93e0f49ef952d8a2ecd068fecd192a97885b7 \ + --hash=sha256:8d47c691765cf497aaeed4954d226568563f1b3b74ff61139f2d77876717084b \ + --hash=sha256:8e3a839bf36ec03b4315dc924d36dcde5444a50066f1c10f8290293c0427b46a \ + --hash=sha256:9348a0137568c45601b031a8d118275069435f151cbb77e6a08a27e8125f59d4 \ + --hash=sha256:a08e2a8a039a3f72afb67a6668180f09fddaa38fe0d21f13212b4aba4b5d2451 \ + --hash=sha256:a218670b26fb1bc74796458d97bcab072765f9b524f95b2fccad70158feb8b17 \ + --hash=sha256:a22a4bc0937856263df8b94f2f2781b33dd7f876f787ed746608e06902d691a5 \ + --hash=sha256:a7bbe7109ab6198688b7079cbad5a8c22de4d47c4880d8e4847520a83b0d1b68 \ + --hash=sha256:a92aff08e23d281c69835e4a47b80569242a504095ef6a6223c1f6bb8883431d \ + --hash=sha256:b34d3bde38eba66190b215bae441646330f8e9da05c29e4b5dd3e41bde701098 \ + --hash=sha256:b903afaa5df66d50fc38672ad095806443b05f202c792694f3a604ead7c6ea6e \ + --hash=sha256:be686bf625aa7b9bada18defb3a3ea3981c1099697239788ff111d87f04cd263 \ + --hash=sha256:c0423393e4a07ff6fea08feb44153302dd261d0551cc3b538ea7a5dc853af43a \ + --hash=sha256:c318eda14f6627966997a7d8c374a87d084a94e4e38e9abbe97395c215830e0c \ + --hash=sha256:c3b78eff5968a1889a0f3bc81ca57e1e19b75f664d9c61a42a604bf9d8402aae \ + --hash=sha256:c73268cf557e688efb60f1ccbc7376f7e18cd8e2acae9e663e98b194c40c1a2d \ + --hash=sha256:c751c1c93955b7a84c06794df46f1cec93e18610dcd5ab7d08e89a81df70a849 \ + --hash=sha256:ce42275097512d9e4e4a39aade58ef2b3798a93aa3026566b7892177c266f735 \ + --hash=sha256:cf3bf0ce511b833f7bc5f5bb3127ba731e97222023a444b7359f3a22e2a3b463 \ + --hash=sha256:da410b70a7ab8eb524112f037a7a35da7128b33d484f7671a264a4c224ac131d \ + --hash=sha256:e675a3ad4732b92d72e4d24009707e923cab76b0d088e5054914f11a797ebe44 \ + --hash=sha256:e82c3d5e44e969c217827b780ed8faf7ac4c53f934ae9238872e749fa531f7c9 \ + --hash=sha256:edfe6d3916e915ada9acc4e48f6dafca7efdbad2e6283db6fd9385a1b23055f1 \ + --hash=sha256:f094742275586cdd6b1a03655ccff3b24b2610c3af76f810356c4c71d24a2a6c \ + --hash=sha256:f208c3b58a6df3b239e0bb130e13bc7487ed14f39a9ff357b6415e3f6339b560 \ + --hash=sha256:f43f5aef2a13d4d56adadae5720d1fed4c1356c993eda8b59dace4b5983843c1 + # via influxdb3-python +python-dateutil==2.9.0.post0 \ + --hash=sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3 \ + --hash=sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427 + # via influxdb3-python +reactivex==4.0.4 \ + --hash=sha256:0004796c420bd9e68aad8e65627d85a8e13f293de76656165dffbcb3a0e3fb6a \ + --hash=sha256:e912e6591022ab9176df8348a653fe8c8fa7a301f26f9931c9d8c78a650e04e8 + # via influxdb3-python +six==1.17.0 \ + --hash=sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274 \ + --hash=sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81 + # via python-dateutil +typing-extensions==4.12.2 \ + --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \ + --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8 + # via reactivex +urllib3==2.3.0 \ + --hash=sha256:1cee9ad369867bfdbbb48b7dd50374c0967a0bb7710050facf0dd6911440e3df \ + --hash=sha256:f8c5449b3cf0861679ce7e0503c7b44b5ec981bec0d1d3795a07f1ba96f0204d + # via influxdb3-python + +# The following packages are considered to be unsafe in a requirements file: +setuptools==75.8.0 \ + --hash=sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6 \ + --hash=sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3 + # via influxdb3-python diff --git a/Library/Homebrew/formula-analytics/transform_analytics_to_counts.json b/Library/Homebrew/formula-analytics/transform_analytics_to_counts.json new file mode 100644 index 0000000000000..f4617ab4b273d --- /dev/null +++ b/Library/Homebrew/formula-analytics/transform_analytics_to_counts.json @@ -0,0 +1,15 @@ +[ + { + "apiVersion": "influxdata.com/v2alpha1", + "kind": "Task", + "metadata": { + "name": "elegant-archimedes-94d001" + }, + "spec": { + "every": "1h", + "name": "Transform analytics to counts", + "query": "errors =\n from(bucket: \"analytics\")\n |> range(start: -task.every)\n |> filter(fn: (r) => r._measurement == \"build_error\" and r._field == \"package\")\n |> reduce(\n fn: (r, accumulator) =>\n ({\n pkg: r._value,\n _value: 1 + accumulator._value,\n _time: r._time,\n _field: r._field,\n _measurement: r._measurement,\n }),\n identity: {\n pkg: \"\",\n _value: 0,\n _time: now(),\n _measurement: \"\",\n _field: \"\",\n },\n )\n |> keep(\n columns: [\n \"_value\",\n \"pkg\",\n \"_time\",\n \"_measurement\",\n \"_field\",\n ],\n )\n |> to(bucket: \"analytics_downsampled\")\n\nformula =\n from(bucket: \"analytics\")\n |> range(start: -task.every)\n |> filter(fn: (r) => r._measurement == \"formula_install\" and r._field == \"package\")\n |> reduce(\n fn: (r, accumulator) =>\n ({\n pkg: r._value,\n _value: 1 + accumulator._value,\n _time: r._time,\n _field: r._field,\n _measurement: r._measurement,\n }),\n identity: {\n pkg: \"\",\n _value: 0,\n _time: now(),\n _measurement: \"\",\n _field: \"\",\n },\n )\n |> keep(\n columns: [\n \"_value\",\n \"pkg\",\n \"_time\",\n \"_measurement\",\n \"_field\",\n ],\n )\n |> to(bucket: \"analytics_downsampled\")\n\nreqFormula =\n from(bucket: \"analytics\")\n |> range(start: -task.every)\n |> filter(\n fn: (r) =>\n r._measurement == \"formula_install\" and r._field == \"package\" and r.on_request\n ==\n \"true\",\n )\n |> reduce(\n fn: (r, accumulator) =>\n ({\n pkg: r._value,\n _value: 1 + accumulator._value,\n _time: r._time,\n _field: r._field,\n _measurement: r._measurement,\n measure_col: \"formula_install_on_request\",\n }),\n identity: {\n pkg: \"\",\n _value: 0,\n _time: now(),\n _measurement: \"\",\n measure_col: \"formula_install_on_request\",\n _field: \"\",\n },\n )\n |> keep(\n columns: [\n \"_value\",\n \"pkg\",\n \"_time\",\n \"_measurement\",\n \"measure_col\",\n \"_field\",\n ],\n )\n |> to(bucket: \"analytics_downsampled\", measurementColumn: \"measure_col\")\n\ncasks =\n from(bucket: \"analytics\")\n |> range(start: -task.every)\n |> filter(fn: (r) => r._measurement == \"cask_install\" and r._field == \"package\")\n |> reduce(\n fn: (r, accumulator) =>\n ({\n pkg: r._value,\n _value: 1 + accumulator._value,\n _time: r._time,\n _field: r._field,\n _measurement: r._measurement,\n }),\n identity: {\n pkg: \"\",\n _value: 0,\n _time: now(),\n _measurement: \"\",\n _field: \"\",\n },\n )\n |> keep(\n columns: [\n \"_value\",\n \"pkg\",\n \"_time\",\n \"_measurement\",\n \"_field\",\n ],\n )\n |> to(bucket: \"analytics_downsampled\")\n\n// Agregate analytics\noses =\n from(bucket: \"analytics\")\n |> range(start: -task.every)\n |> filter(\n fn: (r) =>\n (r._measurement == \"formula_install\" or r._measurement == \"cask_install\")\n and\n r._field == \"os_name_and_version\",\n )\n |> reduce(\n fn: (r, accumulator) =>\n ({\n os: r._value,\n _value: 1 + accumulator._value,\n _time: r._time,\n _field: r._field,\n _measurement: r._measurement,\n measure_col: \"os_versions\",\n }),\n identity: {\n os: \"\",\n _value: 0,\n _time: now(),\n _measurement: \"\",\n measure_col: \"os_versions\",\n _field: \"\",\n },\n )\n |> keep(\n columns: [\n \"_value\",\n \"os\",\n \"_time\",\n \"_measurement\",\n \"measure_col\",\n \"_field\",\n ],\n )\n |> to(bucket: \"analytics_downsampled\", measurementColumn: \"measure_col\")", + "status": "active" + } + } +] \ No newline at end of file diff --git a/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/formula_analytics.rbi b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/formula_analytics.rbi new file mode 100644 index 0000000000000..10517cad8a1b1 --- /dev/null +++ b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/formula_analytics.rbi @@ -0,0 +1,61 @@ +# typed: true + +# DO NOT EDIT MANUALLY +# This is an autogenerated file for dynamic methods in `Homebrew::DevCmd::FormulaAnalytics`. +# Please instead update this file by running `bin/tapioca dsl Homebrew::DevCmd::FormulaAnalytics`. + + +class Homebrew::DevCmd::FormulaAnalytics + sig { returns(Homebrew::DevCmd::FormulaAnalytics::Args) } + def args; end +end + +class Homebrew::DevCmd::FormulaAnalytics::Args < Homebrew::CLI::Args + sig { returns(T::Boolean) } + def all_core_formulae_json?; end + + sig { returns(T::Boolean) } + def brew_command_run?; end + + sig { returns(T::Boolean) } + def brew_command_run_options?; end + + sig { returns(T::Boolean) } + def brew_test_bot_test?; end + + sig { returns(T::Boolean) } + def build_error?; end + + sig { returns(T::Boolean) } + def cask_install?; end + + sig { returns(T.nilable(String)) } + def days_ago; end + + sig { returns(T::Boolean) } + def homebrew_devcmdrun_developer?; end + + sig { returns(T::Boolean) } + def homebrew_os_arch_ci?; end + + sig { returns(T::Boolean) } + def homebrew_prefixes?; end + + sig { returns(T::Boolean) } + def homebrew_versions?; end + + sig { returns(T::Boolean) } + def install?; end + + sig { returns(T::Boolean) } + def install_on_request?; end + + sig { returns(T::Boolean) } + def json?; end + + sig { returns(T::Boolean) } + def os_version?; end + + sig { returns(T::Boolean) } + def setup?; end +end diff --git a/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/generate_analytics_api.rbi b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/generate_analytics_api.rbi new file mode 100644 index 0000000000000..63461ae03c4d1 --- /dev/null +++ b/Library/Homebrew/sorbet/rbi/dsl/homebrew/dev_cmd/generate_analytics_api.rbi @@ -0,0 +1,13 @@ +# typed: true + +# DO NOT EDIT MANUALLY +# This is an autogenerated file for dynamic methods in `Homebrew::DevCmd::GenerateAnalyticsApi`. +# Please instead update this file by running `bin/tapioca dsl Homebrew::DevCmd::GenerateAnalyticsApi`. + + +class Homebrew::DevCmd::GenerateAnalyticsApi + sig { returns(Homebrew::DevCmd::GenerateAnalyticsApi::Args) } + def args; end +end + +class Homebrew::DevCmd::GenerateAnalyticsApi::Args < Homebrew::CLI::Args; end diff --git a/Library/Homebrew/sorbet/tapioca/config.yml b/Library/Homebrew/sorbet/tapioca/config.yml index 707b1bafb61f3..d4e2f10cdd171 100644 --- a/Library/Homebrew/sorbet/tapioca/config.yml +++ b/Library/Homebrew/sorbet/tapioca/config.yml @@ -35,4 +35,7 @@ gem: - unicode-display_width - unicode-emoji - yard-sorbet + # The tapioca generated gem is not correct or sufficient for pycall + # so we need to generate our own: + - pycall prerequire: sorbet/tapioca/prerequire.rb diff --git a/Library/Homebrew/test/dev-cmd/formula-analytics_spec.rb b/Library/Homebrew/test/dev-cmd/formula-analytics_spec.rb new file mode 100644 index 0000000000000..76704d290958f --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/formula-analytics_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "cmd/shared_examples/args_parse" +require "dev-cmd/formula-analytics" + +RSpec.describe Homebrew::DevCmd::FormulaAnalytics do + it_behaves_like "parseable arguments" +end diff --git a/Library/Homebrew/test/dev-cmd/generate-analytics-api_spec.rb b/Library/Homebrew/test/dev-cmd/generate-analytics-api_spec.rb new file mode 100644 index 0000000000000..1253d97206a82 --- /dev/null +++ b/Library/Homebrew/test/dev-cmd/generate-analytics-api_spec.rb @@ -0,0 +1,8 @@ +# frozen_string_literal: true + +require "cmd/shared_examples/args_parse" +require "dev-cmd/generate-analytics-api" + +RSpec.describe Homebrew::DevCmd::GenerateAnalyticsApi do + it_behaves_like "parseable arguments" +end