Updated the import scripts to properly resolve all geometry values fo…

…r nodes, ways, relations, and changesets.

Rakefile

@@ -47,17 +47,17 @@ namespace :import do
 
 	desc "Import PBF File (Nodes, Ways, Relations)"
 	task :pbf do
-		puts window.run_mongo_import
+		window.run_mongo_import
 	end
 
 	desc "Import Changesets"
 	task :changesets do
-		puts window.changeset_import
+		window.changeset_import
 	end
 
 	desc "Import Users"
 	task :users do
-		puts window.user_import
+		window.user_import
 	end
 end
 

analysis_windows/nic_test.yml

@@ -22,7 +22,7 @@ start_date: '2010-01-01'
 end_date:   '2010-12-31'
 
 #Bounding Box:
-bbox: '-86.33,12.1,-86.30,12.2' #Expecting 4 parameters
+bbox: '-86.354212,12.084238,-86.133199,12.191113' #Expecting 4 parameters
 
 #Or:
 max_lon: 

import_scripts/import_analysis_window.rb

@@ -32,16 +32,16 @@ def initialize(args = {})
 
 		begin
 			@global_config = YAML.load_file('config.yml')
-			puts global_config
 		rescue
 			raise IOError.new("Error loading global configuration file")
 		end
+		connect_to_database
 	end
 
 	#Calls the Singleton Database Connection for the specific database
 	def connect_to_database
 		#Open Database Connection
-		puts "Connecting to: #{config['database']} Mongo Database"
+		puts "Connecting to: #{config['database']} Mongo Database\n"
 		DatabaseConnection.new(database: config['database'])
 	end
 
@@ -68,25 +68,23 @@ def run_osm_history_splitter
 	end
 
 	def run_mongo_import
-		connect_to_database
 		conn = OSMPBF.new(end_date: time_frame.end)
 		conn.open_parser("import_scripts/temp.osm.pbf")
 		puts conn.file_stats
 
 		#Import Nodes, Ways, Relations
 		conn.parse_to_collection(object_type="nodes", lim=nil)
 
-		#TODO: Build Node Indexes so Ways can get their geometries
 		conn.parse_to_collection(object_type="ways", lim=nil)
 
-		#TODO: Build Node, Way Indexes so Relations can get their members
 		conn.parse_to_collection(object_type="relations", lim=nil)
 	end
 
 	def changeset_import
 		changeset_import = ChangesetImport.new
 		puts "Importing #{changeset_import.distinct_changeset_ids.length} changesets"
 		changeset_import.import_changeset_objects
+		changeset_import.add_indexes
 	end
 
 	def user_import

import_scripts/osm_api/import_changesets.rb

@@ -49,6 +49,12 @@ def import_changeset_objects
     end
   end
 
+  def add_indexes
+    DatabaseConnection.database['changesets'].ensure_index( id: 1 )
+    DatabaseConnection.database['changesets'].ensure_index( user: 1 )
+    DatabaseConnection.database['changesets'].ensure_index( geometry: "2dsphere")
+  end
+
   def convert_osm_api_to_domain_object_hash(osm_api_hash)
     data = osm_api_hash[:osm][:changeset]
 

import_scripts/pbf_to_mongo.rb

@@ -1,20 +1,9 @@
-'''
-The OSMGeoJSONMongo reads a PBF file and wll create the appropriate collections in the database.
-
-PBF Parser from: https://github.com/planas/pbf_parser
-
-//These two are for parsing PBF: (mac oriented)
-brew install protobuf-c
-gem install pbf_parser
-'''
-
-#Require the PBF Parser -- Subject to change to just 'pbf_parser' with new release
 require 'pbf_parser'
 
 class OSMPBF
 	require 'date'
 
-	attr_reader :parser, :missing_nodes, :n_count, :w_count, :r_count, :file, :nodes, :ways, :end_date
+	attr_reader :parser, :missing_nodes, :n_count, :w_count, :r_count, :file, :nodes, :ways, :end_date, :memory_only
 
 	def initialize(args={})
 
@@ -30,7 +19,11 @@ def initialize(args={})
 		@nodes = {}
 		@ways  = {}
 
+		@memory_only = args[:memory_only] || false
+
+		puts "---------------------------------------"
 		puts "Only parsing data up to #{end_date}"
+		puts "---------------------------------------"
 	end
 
 
@@ -46,6 +39,7 @@ def reset_parser
 		@parser = PbfParser.new(file)
 	end
 
+	#Get stats on the PBF file.
 	def file_stats
 		test_parser = PbfParser.new(file)
 		while test_parser.next
@@ -62,6 +56,7 @@ def file_stats
 		puts "Nodes: #{n_count}, Ways: #{w_count}, Relations: #{r_count}"
 	end
 
+	#Convert the Timestmap to an instance of Time
 	def timestamp_to_date(timestamp)
 		Time.at(timestamp/1000).utc #This is a time instance, it should go straight ot ruby
 	end
@@ -70,7 +65,7 @@ def add_node(node)
 		node[:created_at] = timestamp_to_date(node[:timestamp])
 		this_node = Node.new(node)
 
-		this_node.save!		#Save to Database
+		this_node.save! unless memory_only #Save to Database
 		this_node.mem_save  #Save to Memory
 	end
 
@@ -82,27 +77,30 @@ def add_way(way)
 
 		this_way = Way.new(way)
 
-		this_way.save!      #Save to Database
+
+		this_way.save! unless memory_only #Save to Database
 		this_way.mem_save   #Save to Memory
 	end
 
 
 
 	def add_relation(relation)
 		relation[:created_at] = timestamp_to_date(relation[:timestamp])
-		relation[:nodes] = relation[:members][:nodes]
-		relation[:ways]  = relation[:members][:ways]
+		relation[:nodes] = relation[:members][:nodes].collect{|n| n[:id].to_s}
+		relation[:ways]  = relation[:members][:ways].collect{|w| w[:id].to_s}
 
 		relation.delete :members
 
 		this_rel = Relation.new(relation)
+
 		this_rel.save!
 	end
 
 
 	def parse_to_collection(object_type, lim=nil)
 		start_time = Time.now
     	puts "Started #{object_type} import at: #{start_time}"
+    	puts "-----------------------------------------------\n"
 		reset_parser #Reset the parser because 'seek' does not work
 
 		@missing_nodes = 0
@@ -135,9 +133,9 @@ def parse_to_collection(object_type, lim=nil)
 							end
 						end
 					end
-					if index%2000==0
+					if index%5000==0
 						puts "Processed #{index} of #{count} #{object_type}"
-						if index%10000==0
+						if index%1000==0
         			rate = index/(Time.now() - start_time) #Tweets processed / seconds elapsed
         			mins = (count-index) / rate / 60         #minutes left = tweets left * seconds/tweet / 60
         			hours = mins / 60
@@ -148,12 +146,13 @@ def parse_to_collection(object_type, lim=nil)
 			end
 		end
 
-		puts "Adding the appropriate indexes: id, changeset, geometry"
+		puts "Adding the appropriate indexes: id, changeset, geometry\n"
+		puts "=======================================================\n\n"
 		begin
-			eval %Q{DatabaseConnection.database[#{object_type}.to_s].ensure_index( "id" => 1 ) }
-			eval %Q{DatabaseConnection.database[#{object_type}.to_s].ensure_index( "changeset" => 1 ) }
-			eval %Q{DatabaseConnection.database[#{object_type}.to_s].ensure_index( "geometry" => "2dsphere") }
-		rescue
+			DatabaseConnection.database[object_type.to_sym].ensure_index( id: 1 )
+			DatabaseConnection.database[object_type.to_sym].ensure_index( changeset: 1 )
+			DatabaseConnection.database[object_type.to_sym].ensure_index( geometry: "2dsphere")
+		rescue => e
 			puts "Error creating index"
 			p $!
 		end

models/DomainObjects.rb

@@ -1,20 +1,15 @@
-# 
-# Base Domain Object Class.
-# 
-# 
-#
-
 require_relative '../modules/domain_objects/osm_to_mongo'
 require_relative '../modules/domain_objects/osm_geo'
 
 class OSMObject
 
 	include OSMongoable::OSMObject
+	include OSMGeo::OSMObject
 
-	attr_reader :id, :uid, :user, :created_at, :tags
+	attr_reader :id, :uid, :user, :created_at, :tags, :geometry
 
 	def initialize(args)
-		@id         ||= args[:id]
+		@id         ||= args[:id].to_s
 		@uid    	||= args[:uid]
 		@user  		||= args[:user]
 		@created_at ||= args[:created_at]
@@ -47,7 +42,7 @@ class Way < OSMObject
 	attr_reader :nodes, :version, :changeset, :missing_nodes
 
 	def initialize(args)
-		@nodes = args[:nodes]
+		@nodes 		||= args[:nodes].collect{|node| node.to_s}
 		@version    ||= args[:version]
 		@changeset  ||= args[:changeset]
 		super(args)
@@ -58,7 +53,7 @@ class Relation < OSMObject
 
 	include OSMongoable::Relation
 
-	attr_reader :nodes, :ways, :version, :changeset
+	attr_reader :nodes, :ways, :version, :changeset, :missing_nodes, :missing_ways
 
 	def initialize(args)
 		@nodes = args[:nodes]
@@ -80,10 +75,10 @@ def initialize(args)
 		@comment   = args[:comment]
 		@closed_at = args[:closed_at]
 		@open      = args[:open]
-		@min_lat   = args[:min_lat]
-		@max_lat   = args[:max_lat]
-		@min_lon   = args[:min_lon]
-		@max_lon   = args[:max_lon]
+		@min_lat   = args[:min_lat].to_f
+		@max_lat   = args[:max_lat].to_f
+		@min_lon   = args[:min_lon].to_f
+		@max_lon   = args[:max_lon].to_f
 		super(args)
 	end
 

modules/domain_objects/osm_geo.rb

@@ -3,6 +3,14 @@ module OSMGeo
 	require 'rgeo'
 	Factory = RGeo::Geographic.simple_mercator_factory
 
+	module OSMObject
+
+		def geojson_geometry
+			@geometry ||= get_geojson_geometry
+		end
+
+	end
+
 	module Node
 
 		def point