Implement support for GMaps-based geolocation

gousiosg · Sep 27, 2018 · 8f2fdc3 · 8f2fdc3
1 parent a6caa2d
commit 8f2fdc3
Show file tree

Hide file tree

Showing 4 changed files with 53 additions and 12 deletions.
diff --git a/bin/ght-geolocate b/bin/ght-geolocate
@@ -20,8 +20,10 @@ class GHTGeolocate < GHTorrent::Command
 Geolocates the given address using GHTorrent's APIs and config.
     BANNER
 
-    options.opt :engine, 'Geolocation engine to use (osm or bing)',
+    options.opt :engine, 'Geolocation engine to use (osm, bing or gmaps)',
                 :type => String
+    options.opt :cache, 'Use cached entry?',
+                :type => TrueClass, :default => false
   end
 
   def validate
@@ -31,7 +33,7 @@ Geolocates the given address using GHTorrent's APIs and config.
 
   def go
     @settings = override_config(@settings, :geolocation_service, options[:engine])
-    pp geolocate(location: ARGV[0], from_cache: false)
+    pp geolocate(location: ARGV[0], from_cache: options[:cache])
   end
 
 end

diff --git a/config.yaml.tmpl b/config.yaml.tmpl
@@ -82,9 +82,11 @@ logging:
   file: "stdout"
 
 geolocate:
-    # osm (open street maps) or bing
+    # osm (open street maps), bing or gmaps
     service: osm
     # Time to wait between geo location API requests
     wait: 2
-    # The following are only necessary if bing is used
+    # The following is only necessary if bing is used
     bing_key: foobar
+    # The following is only necessary if gmaps is used
+    gmaps_key: barfoo
diff --git a/lib/ghtorrent/geolocator.rb b/lib/ghtorrent/geolocator.rb
@@ -92,6 +92,37 @@ def parse_geolocation_result(location, geocoded)
   end
 end
 
+module GHTorrent
+  module Geolocator
+    module GMaps
+
+      def format_url(location)
+        URI.escape("https://maps.googleapis.com/maps/api/geocode/json?key=#{config(:geolocation_gmaps_key)}&address=#{location}")
+      end
+
+      def parse_geolocation_result(location, geocoded)
+        details = geocoded['results'].first
+        city_hash = details['address_components'].find{|x| x['types'].include? 'locality'}
+        country_hash = details['address_components'].find{|x| x['types'].include? 'country'}
+        admin_area_hash = details['address_components'].select{|x| not x['types'].grep(/administrative_area/).empty?}.first
+
+        geo = {
+            :key          => location,
+            :long         => details['geometry']['location']['lng'],
+            :lat          => details['geometry']['location']['lat'],
+            :city         => unless city_hash.nil? then city_hash['long_name'] end,
+            :country      => unless country_hash.nil? then country_hash['long_name'] end,
+            :state        => unless admin_area_hash.nil? then admin_area_hash['long_name'] end,
+            :country_code => unless country_hash.nil? then country_hash['short_name'].downcase end,
+            :status       => :ok
+        }
+
+        geo
+      end
+    end
+  end
+end
+
 module GHTorrent
   module Geolocator
 
@@ -138,10 +169,12 @@ def geolocate(location: nil, wait: config(:geolocation_wait).to_i, from_cache: t
 
       if geo.empty?
 
-        if config(:geolocation_service) == 'osm'
-          self.class.send :include, GHTorrent::Geolocator::OSM
-        else
+        if config(:geolocation_service) == 'gmaps'
+          self.class.send :include, GHTorrent::Geolocator::GMaps
+        elsif config(:geolocation_service) == 'bing'
           self.class.send :include, GHTorrent::Geolocator::Bing
+        else
+          self.class.send :include, GHTorrent::Geolocator::OSM
         end
 
         begin
@@ -151,16 +184,19 @@ def geolocate(location: nil, wait: config(:geolocation_wait).to_i, from_cache: t
           p   = JSON.parse(req.read)
           geo = parse_geolocation_result(location, p)
 
-          info "Successful geolocation request. Location: #{location}, URL: #{url}"
+          info "Successful geolocation request. Location: #{location}"
         rescue StandardError => e
-          warn "Failed geolocation request. URL: #{url}"
+          warn "Failed geolocation request. Location: #{location}"
           geo       = EMPTY_LOCATION
           geo[:key] = location
         ensure
           in_db_geo = persister.find(:geo_cache, {'key' => location}).first
 
           if in_db_geo.nil?
             begin
+
+              geo[:updated_at] = Time.now
+
               persister.store(:geo_cache, geo)
             rescue StandardError => e
               warn "Could not save location #{location} -> #{geo}: #{e.message}"

diff --git a/lib/ghtorrent/settings.rb b/lib/ghtorrent/settings.rb
@@ -39,7 +39,8 @@ module Settings
 
         :geolocation_service => 'geolocation.service',
         :geolocation_wait => 'geolocation.wait',
-        :geolocation_bing_key => 'geolocation.bing_key'
+        :geolocation_bing_key => 'geolocation.bing_key',
+        :geolocation_gmaps_key => 'geolocation.gmaps_key'
     }
 
     DEFAULTS = {
@@ -51,7 +52,6 @@ module Settings
         :amqp_prefetch  => 1,
 
         :sql_url => 'sqlite://github.db',
-        # :sql_url => 'mysql2://ghtorrent:ghtorrent@localhost/ghtorrent',
 
         :mirror_urlbase => 'https://api.github.com/',
         :mirror_persister => 'noop',
@@ -74,7 +74,8 @@ module Settings
 
         :geolocation_service => 'osm',
         :geolocation_wait => '2',
-        :geolocation_bing_key => ''
+        :geolocation_bing_key => '',
+        :geolocation_gmaps_key => ''
     }
 
     def config(key, use_default = true)