diff --git a/tasks/download/fishing_report.rake b/tasks/download/fishing_report.rake deleted file mode 100644 index a5ce45f..0000000 --- a/tasks/download/fishing_report.rake +++ /dev/null @@ -1,39 +0,0 @@ -# frozen_string_literal: true - -namespace :download do - ## - task :fishing_report do |tn| - log :debug, tn, 'started' - - ## use mechanize gem to browse the webpage and click the export link we need - agent = Mechanize.new - - url = 'https://nrm.dfg.ca.gov/FishPlants/' - - log :debug, tn, "reading HTML from #{url}" - - page = agent.get(url) - - log :debug, tn, 'loaded page' - - form = page.forms.first - - button = form.buttons.select { |btn| btn.type == 'submit' }.first - - log :debug, tn, 'clicking Export button' - response = agent.submit(form, button) - - form.add_button_to_query(button) - - log :debug, tn, 'saving to file' - - csv_file = File.join(File.dirname(__FILE__), '../../tmp/ca_fishing.csv') - - File.open(csv_file, 'wb') { |f| f << response.body } - - log :info, tn, 'done' - rescue StandardError => e - log :fatal, tn, e.message - log :fatal, tn, e.backtrace - end -end diff --git a/tasks/import/fishing_report.rake b/tasks/import/fishing_report.rake deleted file mode 100644 index c839963..0000000 --- a/tasks/import/fishing_report.rake +++ /dev/null @@ -1,41 +0,0 @@ -# frozen_string_literal: true - -namespace :import do - task :fishing_report do |tn| - log :debug, tn, 'start' - - csv_file = File.join(File.dirname(__FILE__), '../../tmp/ca_fishing.csv') - - i = 0 - File.foreach(csv_file) do |line| - i += 1 - next if i == 1 - - # p line: line - csv_data = CSV.parse(line, col_sep: ',', headers: false) - row = csv_data[0] - hash = { date: row[0], county: row[1], name: row[2], specie: row[3], coordinates: [row[4], row[5]], unit_id: row[6] } - hash[:date] = DateTime.strptime(hash[:date], '%m/%d/%Y %H:%M:%S %p') - - location = FR::Location.where(unit_id: hash[:unit_id], date: hash[:date]).first - location = FR::Location.new if location.nil? - - location.state = 'CA' - location.date = hash[:date] - location.county = hash[:county] - location.name = hash[:name] - location.specie = hash[:specie] - location.coordinates = [hash[:coordinates][0], hash[:coordinates][1]] - location.unit_id = hash[:unit_id] - location.save! - rescue StandardError => e - log :warn, tn, e.message - log :warn, tn, e.backtrace - end - - log :info, tn, 'done' - rescue StandardError => e - log :fatal, tn, e.message - log :fatal, tn, e.backtrace - end -end diff --git a/tasks/import/geofences.rake b/tasks/import/geofences.rake index a527f84..8221681 100644 --- a/tasks/import/geofences.rake +++ b/tasks/import/geofences.rake @@ -1,6 +1,7 @@ # frozen_string_literal: true namespace :import do + ## task :geofences do |tn| log :debug, tn, 'start' diff --git a/tasks/run.rake b/tasks/run.rake index 960737f..cbe1815 100644 --- a/tasks/run.rake +++ b/tasks/run.rake @@ -1,8 +1,8 @@ # frozen_string_literal: true task run: [ - 'download:fishing_report', - 'import:fishing_report', + 'states:california:download', + 'states:california:import', 'notifications:interests' ] do end diff --git a/tasks/states/california/download.rake b/tasks/states/california/download.rake new file mode 100644 index 0000000..d87e467 --- /dev/null +++ b/tasks/states/california/download.rake @@ -0,0 +1,42 @@ +# frozen_string_literal: true + +namespace :states do + ## + namespace :california do + ## + task :download do |tn| + log :debug, tn, 'started' + + ## use mechanize gem to browse the webpage and click the export link we need + agent = Mechanize.new + + url = 'https://nrm.dfg.ca.gov/FishPlants/' + + log :debug, tn, "reading HTML from #{url}" + + page = agent.get(url) + + log :debug, tn, 'loaded page' + + form = page.forms.first + + button = form.buttons.select { |btn| btn.type == 'submit' }.first + + log :debug, tn, 'clicking Export button' + response = agent.submit(form, button) + + form.add_button_to_query(button) + + log :debug, tn, 'saving to file' + + csv_file = File.join(File.dirname(__FILE__), '../../../tmp/ca_fishing.csv') + + File.open(csv_file, 'wb') { |f| f << response.body } + + log :info, tn, 'done' + rescue StandardError => e + log :fatal, tn, e.message + log :fatal, tn, e.backtrace + end + end +end diff --git a/tasks/states/california/import.rake b/tasks/states/california/import.rake new file mode 100644 index 0000000..e1e7811 --- /dev/null +++ b/tasks/states/california/import.rake @@ -0,0 +1,45 @@ +# frozen_string_literal: true + +namespace :states do + ## + namespace :california do + ## + task :import do |tn| + log :debug, tn, 'start' + + csv_file = File.join(File.dirname(__FILE__), '../../../tmp/ca_fishing.csv') + + i = 0 + File.foreach(csv_file) do |line| + i += 1 + next if i == 1 + + # p line: line + csv_data = CSV.parse(line, col_sep: ',', headers: false) + row = csv_data[0] + hash = { date: row[0], county: row[1], name: row[2], specie: row[3], coordinates: [row[4], row[5]], unit_id: row[6] } + hash[:date] = DateTime.strptime(hash[:date], '%m/%d/%Y %H:%M:%S %p') + + location = FR::Location.where(unit_id: hash[:unit_id], date: hash[:date]).first + location = FR::Location.new if location.nil? + + location.state = 'CA' + location.date = hash[:date] + location.county = hash[:county] + location.name = hash[:name] + location.specie = hash[:specie] + location.coordinates = [hash[:coordinates][0], hash[:coordinates][1]] + location.unit_id = hash[:unit_id] + location.save! + rescue StandardError => e + log :warn, tn, e.message + log :warn, tn, e.backtrace + end + + log :info, tn, 'done' + rescue StandardError => e + log :fatal, tn, e.message + log :fatal, tn, e.backtrace + end + end +end