Merge pull request #76 from bmos/patch-1

Feed Check Improvements

Gemfile.lock

@@ -50,7 +50,6 @@ GEM
     ffi (1.15.5)
     forwardable-extended (2.6.0)
     gli (2.21.0)
-    google-protobuf (3.23.4)
     google-protobuf (3.23.4-x86_64-linux)
     http_parser.rb (0.8.0)
     i18n (1.14.1)
@@ -94,11 +93,7 @@ GEM
       activerecord
       logutils (>= 0.6.1)
     mercenary (0.4.0)
-    mini_portile2 (2.8.2)
     minitest (5.18.1)
-    nokogiri (1.15.3)
-      mini_portile2 (~> 2.8.2)
-      racc (~> 1.4)
     nokogiri (1.15.3-x86_64-linux)
       racc (~> 1.4)
     opmlparser (1.0.1)
@@ -159,13 +154,9 @@ GEM
     ruby2_keywords (0.0.5)
     rubyzip (2.3.2)
     safe_yaml (1.0.5)
-    sass-embedded (1.63.6)
+    sass-embedded (1.69.5)
       google-protobuf (~> 3.23)
       rake (>= 13.0.0)
-    sass-embedded (1.63.6-x86_64-linux-gnu)
-      google-protobuf (~> 3.23)
-    sqlite3 (1.6.3)
-      mini_portile2 (~> 2.8.0)
     sqlite3 (1.6.3-x86_64-linux)
     terminal-table (3.0.2)
       unicode-display_width (>= 1.1.1, < 3)
@@ -181,7 +172,6 @@ GEM
     zeitwerk (2.6.8)
 
 PLATFORMS
-  ruby
   x86_64-linux
 
 DEPENDENCIES

tests/feedcheck.rb

@@ -1,66 +1,139 @@
 # frozen_string_literal: true
 
-require 'iniparser'
 require 'faraday'
+require 'iniparser'
 require 'nokogiri'
 require 'uri'
 
-hash = INI.load_file('planet.ini')
-av_dir = 'hackergotchi'
+INI_FILE = 'planet.ini'
+AV_DIR = 'hackergotchi'
 
-avatars = []
+def check_avatar(avatar, av_dir, faraday)
+  return ['_ ', false] if avatar.nil?
 
-hash.each do |key, section|
-  next unless section.is_a?(Hash)
+  [check_url(avatar, faraday)] if avatar.include? '//'
 
-  print ":: #{key} =>  "
-  feed = section['feed'] if section.key?('feed')
+  ["✗\nAvatar not found: hackergotchi/#{avatar} ", true] unless File.file?("#{av_dir}/#{avatar}")
+
+  ['✓ ', false]
+end
+
+def check_url(url, faraday)
+  error_message = '✗ '
+
+  begin
+    res = faraday.get(URI(url))
+  rescue Faraday::ConnectionFailed
+    return ["#{error_message}Connection Failure when trying to access '#{url}' ", true]
+  rescue Faraday::SSLError
+    return ["#{error_message}SSL Error when trying to access '#{url}' ", true]
+  end
+
+  error = "#{error_message}Non successful status code #{res.status} when trying to access '#{url}' "
+  if res.status.to_i.between?(300, 399) && res.headers.key?('location')
+    return ["#{error}. Try using '#{res.headers['location']}' instead", true]
+  end
+
+  [error, true] unless res.status.to_i == 200
+
+  ['✓ ', false]
+end
+
+def check_urls(url_arr, faraday)
+  results = url_arr.map { |url| check_url(url, faraday) }
+  [results.map(&:first).join, results.any?(&:last)]
+end
+
+def parse_xml(feed, faraday)
+  result = ['✗ ', true]
+
+  begin
+    xml = faraday.get(URI(feed))
+  rescue Faraday::ConnectionFailed
+    ["#{result.first}Connection Failure when trying to read XML from '#{feed}' ", true]
+  rescue Faraday::SSLError
+    ["#{result.first}SSL Error when trying to read XML from '#{feed}' ", true]
+  end
+
+  xml_err = Nokogiri::XML(xml.body).errors
+  ["#{result.first}Unusable XML syntax: #{feed}\n#{xml_err} ", true] unless xml_err.empty?
+
+  ['✓ ', false]
+end
+
+def check_unused_files(av_dir, avatars)
+  hackergotchis = Dir.foreach(av_dir).select { |f| File.file?("#{av_dir}/#{f}") }
+  diff = (hackergotchis - avatars)
+
+  [nil, false] if diff.empty? || avatars.empty?
+
+  ["There are unused files in hackergotchis:\n#{diff.join(', ')}", true]
+end
+
+def check_source(key, section, faraday)
+  did_fail = false
+  result = [":: #{key} =>  "]
   avatar = section['avatar'] if section.key?('avatar')
-  url_arr = []
-  url_arr << section['link'] if section.key?('link')
-  url_arr << feed if feed
-  # Check if avatar exists
-  if avatar
-    if avatar.include? '//'
-      url_arr << avatar
-    else
-      unless File.file?("#{av_dir}/#{avatar}")
-        print "✗\nAvatar not found: hackergotchi/#{avatar}"
-        abort
-      end
-      avatars << avatar
-    end
+
+  avatar_result = check_avatar(avatar, AV_DIR, faraday)
+  result << avatar_result.first
+  did_fail |= avatar_result.last
+
+  link = section['link'] if section.key?('link')
+  feed = section['feed'] if section.key?('feed')
+  url_result = check_urls([link, feed], faraday)
+  result << url_result.first
+  did_fail |= url_result.last
+
+  # Only check XML validity if URL checked out ok
+  unless url_result.last
+    xml_result = parse_xml(feed, faraday)
+    result << xml_result.first
+    did_fail |= xml_result.last
   end
-  print '✓ '
-  # Check if URLs return 200 status
-  url_arr.each do |url|
-    res = Faraday.get(URI(url))
-    error = "✗\nNon successful status code #{res.status} when trying to access `#{url}`"
-    if res.status.to_i.between?(300, 399) && res.headers.key?('location')
-      print "#{error}\nTry using `#{res.headers['location']}` instead"
-      abort
-    end
-    unless res.status.to_i == 200
-      print error
-      abort
+
+  [[result.compact.join, did_fail], avatar]
+end
+
+planet_srcs = INI.load_file(INI_FILE)
+did_any_fail = false
+error_messages = []
+avatars = ['default.png']
+
+faraday = Faraday.new(request: { open_timeout: 10 }) do |f|
+  f.adapter :net_http
+end
+
+queue = Queue.new
+planet_srcs.each do |key, section|
+  queue.push([key, section]) if ARGV.empty? || ARGV.include?(key)
+end
+
+workers = (0...3).map do
+  Thread.new do
+    until queue.empty?
+      key, section = queue.pop
+      next unless section.is_a?(Hash)
+
+      res, avatar = check_source(key, section, faraday)
+      avatars << avatar
+      puts res.first if res.first
+      error_messages << res.first if res.last
+      did_any_fail ||= res.last
     end
   end
-  print '✓ '
-  # Check is the XML actually parses as XML
-  xml = Faraday.get(URI(feed)).body
-  xml_err = Nokogiri::XML(xml).errors
-  unless xml_err.empty?
-    print "✗\nUnusable XML syntax: #{feed}\n#{xml_err}"
-    abort
-  end
-  puts '✓ '
+end
+workers.each(&:join)
+
+unused_files_result = check_unused_files(AV_DIR, avatars)
+if unused_files_result.last
+  error_messages << unused_files_result.first
+  puts "[WARNING] #{unused_files_result.first}"
 end
 
-avatars << 'default.png'
-avatars.uniq!
-hackergotchis = Dir.foreach(av_dir).select { |f| File.file?("#{av_dir}/#{f}") }
-diff = (hackergotchis - avatars).sort
-unless diff.empty?
-  print "There are unused files in hackergotchis:\n#{diff.join(', ')}"
+if did_any_fail
+  puts "[ERROR] Summary"
+  puts error_messages
   abort
 end
+puts 'All feeds passed checks!'