Fix provider updaters

app/lib/region_area.rb

@@ -0,0 +1,20 @@
+# Define the region area mappings in a field called `prefixes`
+# Example:
+# include RegionArea
+# def initialize
+#   super
+#   @prefixes = {
+#       'us' => 'US',
+#       'eu' => 'EU',
+#   }
+# end
+module RegionArea
+  UNKNOWN = 'UNKNOWN'
+  def extract_region_area(region)
+    @prefixes.each do |prefix, region_area|
+      return region_area if region.start_with?(prefix)
+    end
+    puts "WARNING: Could not match region `#{region}` to a region area. Check `PREFIXES` in `#{self.class}`!"
+    UNKNOWN
+  end
+end

app/models/provider_updater.rb

@@ -2,36 +2,35 @@
 
 class ProviderUpdater < ActiveJob::Base
   queue_as :default
-  
+
   class Error < StandardError; end
-  
+
   def self.providers
     load_providers
     descendants
   end
-  
+
   def self.update_providers
     providers.map(&:perform_later)
   end
-    
+
   private
   def self.load_providers
     Dir[Rails.root + 'app/provider_updaters/*.rb'].each do |updater|
       require updater
     end
   end
-  
+
   def logger
     Rails.logger
   end
-  
+
   # Parse SLA information from an SLA document
   #
   # This method will try to get SLA information from the document at the 
   # specified URI. Currently, it is _very_ simple and will only search for a
   # phrase like "at least <nn.nn>%" to get a basic availability SLA.
   def extract_sla(uri, pattern = %r{at least (\d+(?:\.\d+)?)%}im)
-
     result = { uri: uri.to_s }
 
     doc = Nokogiri::HTML(open(uri))
@@ -40,7 +39,6 @@ def extract_sla(uri, pattern = %r{at least (\d+(?:\.\d+)?)%}im)
       # TODO: Extract more info from SLA.
       # maybe we can even get some decent conditions from the SLA text.
     end
-
     result
   end
 end

app/provider_updaters/amazon_updater.rb

@@ -1,135 +1,74 @@
 class AmazonUpdater < ProviderUpdater
+  include RegionArea
+
+  def initialize
+    super
+    @prefixes = {
+        'us' => 'US',
+        'eu' => 'EU',
+        'ap' => 'ASIA',
+        'sa' => 'SA',
+    }
+  end
+
   def perform
-    provider = update_provider
-    update_compute(provider)
-    update_storage(provider)
+    @provider = Provider.find_or_create_by(name: 'Amazon')
+    update_compute_batch
   end
 
   private
 
-  # crawls all us-east-1 EC2 compute instances and their prices
-  def update_compute(provider)
-    response = make_request('https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js')
-    pricelist = parse_callback(response.body)
-
-    provider.more_attributes['pricelist'][:compute] = pricelist
-    provider.save!
-
-    pricelist['config']['regions'].each do |region_json|
-      region = region_json['region']
-
-      region_json['instanceTypes'].each do |it|
-
-        instance_type = it['type']
-        it['sizes'].each do |s|
-          # Attributes: size, vCPU, ECU, memoryGiB, storageGB, valueColumns
-          resource_id = s['size']
-          resource = provider.resources.find_or_create_by(name: resource_id, region: region)
-
-          resource.resource_type = 'compute'
-          resource.more_attributes['cores'] = s['vCPU']
-          resource.more_attributes['mem_gb'] = BigDecimal.new(s['memoryGiB'])
-          resource.more_attributes['price_per_hour'] = s['valueColumns'].first['prices']['USD']
-          resource.region = region
-          resource.region_area = extract_region_area(region)
-          resource.save!
-        end
+    def update_compute_batch
+      ActiveRecord::Base.transaction do
+        update_compute
       end
     end
 
-  rescue Net::HTTPError, JSON::ParserError => e
-    logger.error "Error, #{e.inspect}"
-  end
-
-
-  def update_storage(provider)
-    response = make_request('http://a0.awsstatic.com/pricing/1/s3/pricing-storage-s3.min.js')
-    pricelist = parse_callback(response.body)
-
-
-    provider.more_attributes['pricelist'][:storage] = pricelist
-    provider.save!
-    pricelist['config']['regions'].each do |region_json|
-      region = region_json['region']
-
-      region_json['tiers'].each do |tier|
-        tier_name = tier['name']
-        tier['storageTypes'].each do |storageType|
-          storage_name = storageType['type']
-          resource_name = tier_name + "_" + storage_name
-          resource = provider.resources.find_or_create_by(name: resource_name, region: region)
-          resource.resource_type = 'storage'
-          resource.region = region
-          resource.more_attributes['price_per_gb'] = storageType['prices']['USD']
-          resource.region_area = extract_region_area(region)
-          resource.save!
+    def update_compute
+      pricelist = parse_callback(get_ec2_pricelist)
+      pricelist['config']['regions'].each do |region_json|
+        region_json['instanceTypes'].each do |family|
+          family['sizes'].each do |instance|
+            create_instance(region_json['region'], instance)
+          end
         end
       end
 
+    rescue => e
+      logger.error "Error updating Amzon EC2 resources, #{e.message}"
+      raise e
     end
 
+    def create_instance(region, instance)
+      resource = @provider.resources.find_or_create_by(name: instance['size'], region: region)
+      resource.resource_type = 'compute'
+      resource.region = region
+      resource.region_area = extract_region_area(region)
+      resource.more_attributes['cores'] = instance['vCPU']
+      resource.more_attributes['mem_gb'] = BigDecimal.new(instance['memoryGiB'])
+      resource.more_attributes['price_per_hour'] = instance['valueColumns'].first['prices']['USD']
+      resource.save!
+    end
 
-  rescue Net::HTTPError, JSON::ParserError => e
-    logger.error "Error, #{e.inspect}"
-  end
-
-
-
-  #update general Amazon provider data
-  def update_provider
-    provider = Provider.find_or_create_by(name: 'Amazon')
-
-    sla_hash = {}
-    sla_hash[:compute] = extract_sla('https://aws.amazon.com/ec2/sla/')
-    sla_hash[:storage] = extract_sla('https://aws.amazon.com/s3/sla/');
-
-    pricelist_hash = {}
-
-    provider.more_attributes['sla'] = sla_hash;
-    provider.more_attributes['pricelist'] = pricelist_hash
-    provider.save!
-    provider
-  end
-
-  # Makes an HTTP-GET request on the specified url and returns the response object
-  def make_request(url)
-    uri = URI(url)
-    http = Net::HTTP.new(uri.host, uri.port)
-    http.use_ssl = (uri.scheme == 'https')
-
-    response = http.get(uri.request_uri)
-
-    # Raises error if response is not 2xx, see http://ruby-doc.org/stdlib-2.1.2/libdoc/net/http/rdoc/Net/HTTPResponse.html#method-i-value
-    response.value
-
-    response
-  end
-
-  # Parses a JS-callback object containing Amazons pricing data
-  def parse_callback(callback)
-    result = ''
-
-    V8::Context.new(timeout: 1000) do |ctx|
-      ctx['result'] = result
-      ctx.eval('function callback(param) { return JSON.stringify(param); }')
-      ctx.eval('result = ' + callback)
-      result = ctx['result']
+    def get_ec2_pricelist
+      uri = URI('https://a0.awsstatic.com/pricing/1/ec2/linux-od.min.js')
+      response = Net::HTTP.get_response(uri)
+      # Raises error if response is not 2xx, see http://ruby-doc.org/stdlib-2.1.2/libdoc/net/http/rdoc/Net/HTTPResponse.html#method-i-value
+      response.value
+      response.body
     end
 
-    JSON.parse(result)
-  end
+    # Parses a JS-callback object containing Amazons pricing data
+    def parse_callback(callback)
+      result = ''
 
+      V8::Context.new(timeout: 1000) do |ctx|
+        ctx['result'] = result
+        ctx.eval('function callback(param) { return JSON.stringify(param); }')
+        ctx.eval('result = ' + callback)
+        result = ctx['result']
+      end
 
-  def extract_region_area(region)
-    if region.downcase().include? 'us'
-      return 'US'
-    elsif region.downcase().include? 'eu'
-      return 'EU'
-    elsif region.downcase().include? 'ap'
-      return 'ASIA'
-    elsif region.downcase().include? 'sa'
-      return 'SA'
+      JSON.parse(result)
     end
-  end
-
 end

app/provider_updaters/atlantic_net_updater.rb

@@ -1,7 +1,29 @@
 class AtlanticNetUpdater < ProviderUpdater
+  include RegionArea
+
+  def initialize
+    super
+    @prefixes = {
+        'US' => 'US',
+        'CA' => 'US',
+        'EU' => 'EU',
+    }
+  end
+
   # Get pricing data from Atlantic.net API
   # see https://www.atlantic.net/docs/api/#describe-plans
   def perform
+    @provider = Provider.find_or_create_by(name: 'Atlantic.net')
+    update_compute_batch
+  end
+
+  def update_compute_batch
+    ActiveRecord::Base.transaction do
+      update_compute
+    end
+  end
+
+  def update_compute
     uri = URI('https://cloudapi.atlantic.net/?Action=describe-plan')
     access_key_id = ENV['ANC_ACCESS_KEY_ID']
     private_key = ENV['ANC_PRIVATE_KEY']
@@ -19,12 +41,12 @@ def perform
 
     request = Net::HTTP::Post.new(uri)
     request.set_form_data(
-      'Version' => version,
-      'ACSAccessKeyId' => access_key_id,
-      'Format' => format,
-      'Timestamp' => timestamp,
-      'Rndguid' => rndguid,
-      'Signature' => Base64.strict_encode64(signature),
+        'Version' => version,
+        'ACSAccessKeyId' => access_key_id,
+        'Format' => format,
+        'Timestamp' => timestamp,
+        'Rndguid' => rndguid,
+        'Signature' => Base64.strict_encode64(signature),
     )
     response = Net::HTTP.start(uri.hostname, uri.port, use_ssl: uri.scheme == 'https') { |http| http.request(request) }
     response.value
@@ -34,23 +56,22 @@ def perform
 
     pricelist = response_json['describe-planresponse']['plans']
 
-    provider = Provider.find_or_create_by(name: 'Atlantic.net')
-    provider.more_attributes['pricelist'] = pricelist
-    provider.more_attributes['sla'] = {
-      uri: 'https://www.atlantic.net/service-policies/cloud-service-level-agreement/',
-      availability: '1'
+    @provider.more_attributes['pricelist'] = pricelist
+    @provider.more_attributes['sla'] = {
+        uri: 'https://www.atlantic.net/service-policies/cloud-service-level-agreement/',
+        availability: '1'
     }
-    provider.save!
+    @provider.save!
 
     pricelist.each_pair do |key, instance_type|
       next unless instance_type['platform'] == platform
       resource_id = instance_type['plan_name']
 
       #TODO: use real regions instead of default EUWEST region
-      regions = ['EUWEST1','USEAST1','USEAST2','USCENTRAL1','USWEST1','CAEAST1']
+      regions = ['EUWEST1', 'USEAST1', 'USEAST2', 'USCENTRAL1', 'USWEST1', 'CAEAST1']
 
       regions.each do |region|
-        resource = provider.resources.find_or_create_by(name: resource_id, region: region)
+        resource = @provider.resources.find_or_create_by(name: resource_id, region: region)
 
         resource.resource_type = 'compute'
         resource.more_attributes['cores'] = instance_type['num_cpu']
@@ -62,18 +83,5 @@ def perform
         resource.save!
       end
     end
-  rescue Net::HTTPError, JSON::ParserError, ProviderUpdater::Error => e
-    logger.error "Error, #{e.inspect}"
   end
-
-
-  private
-    def extract_region_area(region)
-      if (region.downcase().include? 'us') || (region.downcase().include? 'ca')
-        return 'US'
-      elsif region.downcase().include? 'eu'
-        return 'EU'
-      end
-    end
-
 end