diff --git a/Gemfile b/Gemfile index 813c7b652..4e3afacfb 100644 --- a/Gemfile +++ b/Gemfile @@ -4,34 +4,34 @@ ruby "1.9.3" gem 'rails', '3.2.11' -gem 'sqlite3' -gem 'pg' gem 'mysql2' +gem 'pg' +gem 'sqlite3' +gem 'acts_as_commentable' +gem 'acts_as_indexed' +gem 'barista' gem 'bcrypt-ruby' gem 'cancan' -gem 'haml' -gem 'acts_as_commentable' -gem 'will_paginate' -gem 'paperclip', '< 3.0' -gem 'gravatar-ultimate' +gem 'cocoon' gem 'formtastic', '~> 2.1.0' +gem 'formtastic-bootstrap', :git => "git://github.com/katastrophie/formtastic-bootstrap.git" +gem 'gravatar-ultimate' +gem 'haml' gem 'jquery-rails', '~> 1.0.19' -gem 'acts_as_indexed' -gem 'cocoon' -gem 'paper_trail', '2.3.3' +gem 'json' gem 'localized_language_select', '0.2.0', :git => "git://github.com/oneiros/localized_language_select.git" +gem 'nokogiri' +gem 'paperclip', '< 3.0' +gem 'paper_trail', '2.3.3' +gem 'prawn' +gem 'prawn_rails' gem 'ransack' -gem 'transitions', :require => ["transitions", "active_record/transitions"] -gem 'json' -gem 'barista' gem 'ri_cal' -gem 'nokogiri' gem 'settingslogic' +gem 'transitions', :require => ["transitions", "active_record/transitions"] gem 'twitter-bootstrap-rails', :git => "git://github.com/seyhunak/twitter-bootstrap-rails.git", :ref => "5e62b21c8f258010af7f5bc858b89a24f16936a9" -gem 'formtastic-bootstrap', :git => "git://github.com/katastrophie/formtastic-bootstrap.git" -gem 'prawn' -gem 'prawn_rails' +gem 'will_paginate' group :development, :test do gem 'bullet' @@ -40,19 +40,19 @@ group :development, :test do end group :test do - gem 'minitest' gem 'factory_girl_rails', '~> 1.2.0' + gem 'minitest' gem 'turn', :require => false end group :development do gem 'hpricot' - gem 'yaml_db' gem 'shotgun' + gem 'yaml_db' end group :assets do - gem 'sass-rails', " ~> 3.2.0" gem 'coffee-rails', " ~> 3.2.0" + gem 'sass-rails', " ~> 3.2.0" gem 'uglifier' end diff --git a/README.pentabarf.md b/README.pentabarf.md index 3625dce0a..4c4410408 100644 --- a/README.pentabarf.md +++ b/README.pentabarf.md @@ -5,13 +5,15 @@ These notes may help to import data from a pentabarf postgresql database. Using postgresql as a database for frab is still somewhat untested. The pentabarf import is however likely to fail, as pentabarf uses text fields instead of char(255) +Imagemagick needs to be installed as we will convert pjpeg and tiff to png. + ## postgresql installation -install postgresql +Install postgresql ## postgresql setup -* Make it listen on localhost +* make it listen on localhost * create a psql user and grant some access on relations * add a pentabarf entry for the postgresql database to your rails db environment @@ -27,13 +29,22 @@ Make a copy of your postgresql database, as we need to do some changes Grant all permissions on the database copy to the import user account: - psql cccv - select 'grant all on '||schemaname||'.'||tablename||' to bar;' from pg_tables + psql NEWNAME + -- generate the grant statements + select 'grant all on '||schemaname||'.'||tablename||' to frab;' from pg_tables order by schemaname, tablename; + -- copy&paste the generated statements into psql + + -- in case you re-created the NEWNAME copy, re-grant permissions to the user + REVOKE ALL ON SCHEMA public FROM frab; + GRANT ALL ON SCHEMA public TO frab; + REVOKE ALL ON SCHEMA auth FROM frab; + GRANT ALL ON SCHEMA auth TO frab; ## data migration -I had to delete some images, too. +Conference acronyms need to be within /^[a-zA-Z0-9_-]*$/ +Whitespaces are removed automatically, but you need to replace unicode characters manually. -- conference acronyms appear in URLs, they may not contain whitespace in frab: UPDATE conference SET acronym = replace(acronym, ' ', ''); @@ -41,7 +52,6 @@ I had to delete some images, too. ## import - Delete any old mappings from previous imports. Maybe delete the old filess, too. rm tmp/*mappings.yml @@ -49,4 +59,19 @@ Delete any old mappings from previous imports. Maybe delete the old filess, too. RAILS_ENV=production rake db:reset RAILS_ENV=production rake pentabarf:import:all +## testing + +You can check on the barf data like this: + + RAILS_ENV="development" rails console + @p = PentabarfImportHelper.new + @barf = @p.instance_variable_get('@barf') + @barf.select_all("SELECT * FROM conference") + +## privileges + +You maybe want to drop all users to the coordinator role, to start fresh. + +User.all.select { |u| u.role == "admin" or u.role == "orga" }.each { |u| puts "dropping ${u.email}"; u.role = "coordinator"; u.save } + diff --git a/app/controllers/application_controller.rb b/app/controllers/application_controller.rb index 57ec4be45..1cb22575d 100644 --- a/app/controllers/application_controller.rb +++ b/app/controllers/application_controller.rb @@ -49,7 +49,13 @@ def default_url_options end def current_user - @current_user ||= User.find(session[:user_id]) if session[:user_id] + user = nil + # maybe the user got deleted, so lets wrap this in a rescue block + begin + user = User.find(session[:user_id]) if session[:user_id] + rescue + end + @current_user ||= user end def authenticate_user! diff --git a/lib/pentabarf_import_helper.rb b/lib/pentabarf_import_helper.rb index efb312a02..b8d1be559 100644 --- a/lib/pentabarf_import_helper.rb +++ b/lib/pentabarf_import_helper.rb @@ -12,11 +12,11 @@ class PentabarfImportHelper # pentabarf roles are just different ROLE_MAPPING = { + "submitter" => "submitter", + "reviewer" => "reviewer", "comittee" => "coordinator", - "developer" => "admin", "admin" => "orga", - "submitter" => "submitter", - "reviewer" => "reviewer" + "developer" => "admin", } EVENT_STATE_MAPPING = { @@ -54,19 +54,30 @@ def import_conferences conferences = @barf.select_all("SELECT * FROM conference") puts "[ ] importing #{conferences.count} conferences" if DEBUG conference_mapping = create_mappings(:conferences) + conferences.each do |conference| + penta_days = @barf.select_values("SELECT conference_day FROM conference_day + WHERE conference_id = #{conference["conference_id"]} + ORDER BY conference_day ASC") + + fake_days = penta_days + if fake_days.empty? + fake_days << Time.now.ago(1.year) + end new_conference = Conference.create!( :title => conference["title"], # clean up acronyms in pentabarf db first! - :acronym => conference["acronym"], + :acronym => conference["acronym"].gsub(/ /,''), # it's a string like 'Europe/Berlin', 'Berlin' :timezone => conference["timezone"], - #TODO omg timeslots :timeslot_duration => interval_to_minutes(conference["timeslot_duration"]), :default_timeslots => conference["default_timeslots"], :max_timeslots => conference["max_timeslot_duration"], - :feedback_enabled => conference["f_feedback_enabled"], + :feedback_enabled => conference["f_feedback_enabled"] == 'f' ? false : true, + # nowhere to find. just use the conference dates instead.. + :created_at => fake_days.first.to_datetime, + :updated_at => fake_days.last.to_datetime, # TODO ticket server, instead of link? DO TICKET URLS THEY END UP PUBLIC? ) conference_mapping[conference["conference_id"]] = new_conference.id @@ -76,21 +87,31 @@ def import_conferences #puts "+++ %s - %s" % [conference["acronym"], Time.zone] if DEBUG # convert pentabarf days to frab days - penta_days = @barf.select_values("SELECT conference_day FROM conference_day - WHERE conference_id = #{conference["conference_id"]} - ORDER BY conference_day ASC") penta_days.each do |day| day = day.to_datetime - # pentabarf uses a 'day change time', which is set at 04:00 o'clock for ccc congresses + # pentabarf uses a 'day change time', + # which is set at 04:00 o'clock for ccc congresses # so we kind of fix it: - start_date = day.to_datetime.change(:hour => 4, :minute => 0) - end_date = day.since(1.days).to_datetime.change(:hour => 3, :minute => 59) + hour = conference['day_change'].gsub(/:.*/,'').to_i + + start_date = day.to_datetime.change(:hour => hour, :minute => 0) + end_date = day.since(1.days).to_datetime.change(:hour => hour-1, :minute => 59) tmp = Day.new(:conference => new_conference, :start_date => Time.zone.local_to_utc(start_date), :end_date => Time.zone.local_to_utc(end_date)) tmp.save! end - + + # create a dummy cfp for this conference + cfp = CallForPapers.new + cfp.conference = new_conference + cfp.start_date = fake_days.first.to_datetime.ago(3.month) + cfp.end_date = fake_days.first.to_datetime.ago(1.month) + cfp.created_at = cfp.start_date + cfp.updated_at = cfp.end_date + cfp.info_url = conference["homepage"] + cfp.contact_email = conference["email"] + cfp.save! end save_mappings(:conferences) end @@ -222,7 +243,7 @@ def import_accounts :password_confirmation => password ) user.confirmed_at = Time.now - user.role = role ? "submitter" : ROLE_MAPPING[role] + user.role = role ? ROLE_MAPPING[role] : "submitter" user.pentabarf_salt = account["salt"] user.pentabarf_password = account["password"] user.save! @@ -248,21 +269,23 @@ def import_languages def import_links mappings(:people).each do |orig_id, new_id| links = @barf.select_all("SELECT l.title, l.url FROM conference_person as p LEFT OUTER JOIN conference_person_link as l ON p.conference_person_id = l.conference_person_id WHERE p.person_id = #{orig_id}") - puts "[ ] importing #{links.count} links from people" if DEBUG + # puts "[ ] importing #{links.count} links from people" if DEBUG links.each do |link| if link["title"] and link["url"] person = Person.find(new_id) - Link.create(:title => link["title"], :url => link["url"], :linkable => person) + Link.create(:title => truncate_string(link["title"]), + :url => truncate_string(link["url"]), :linkable => person) end end end mappings(:events).each do |orig_id, new_id| links = @barf.select_all("SELECT title, url FROM event_link WHERE event_id = #{orig_id}") - puts "[ ] importing #{links.count} links from events" if DEBUG + #puts "[ ] importing #{links.count} links from events" if DEBUG links.each do |link| if link["title"] and link["url"] event = Event.find(new_id) - Link.create(:title => link["title"], :url => link["url"], :linkable => event) + Link.create(:title => truncate_string(link["title"]), + :url => truncate_string(link["url"]), :linkable => event) end end end @@ -276,6 +299,8 @@ def import_events image = @barf.select_one("SELECT * FROM event_image WHERE event_id = #{event["event_id"]}") image_file = image_to_file(image, "event_id") conference = Conference.find(mappings(:conferences)[event["conference_id"]]) + Time.zone = conference.timezone + new_event = Event.create!( :conference_id => conference.id, :track_id => mappings(:tracks)[event["conference_track_id"]], @@ -287,7 +312,6 @@ def import_events #:state => EVENT_STATE_MAPPING[event["event_state"]], :state => EVENT_PROGRESS_MAPPING[event["event_state_progress"]], :language => event["language"], - #TODO :start_time => start_time(event["conference_day"], event["start_time"]), :room_id => mappings(:rooms)[event["conference_room_id"]], :abstract => event["abstract"], @@ -304,6 +328,8 @@ def import_events end def import_event_ratings + disable_event_callback(EventRating) + event_ratings = @barf.select_all("SELECT * FROM event_rating") puts "[ ] importing #{event_ratings.count} event ratings" if DEBUG event_ratings.each do |rating| @@ -314,12 +340,16 @@ def import_event_ratings :comment => rating["remark"], ) end + + # update in batch + puts "[ ] updating rating average on events" if DEBUG + update_event_average("event_ratings", "average_rating") + enable_event_callbacks(EventRating) end def import_event_feedbacks - # we wan't to update this in batch after all the inserts - Event.skip_callback(:save, :after, :update_conflicts) - EventFeedback.skip_callback(:save, :after, :update_average) + disable_event_callback(EventFeedback) + event_feedbacks = @barf.select_all("SELECT * FROM event_feedback") puts "[ ] importing #{event_feedbacks.count} event feedbacks" if DEBUG event_feedbacks.each do |feedback| @@ -339,7 +369,7 @@ def import_event_feedbacks end # this might happen: if rating.nil? - rating = -99 + rating = 3 end EventFeedback.create!( :event_id => mappings(:events)[feedback["event_id"]], @@ -348,26 +378,11 @@ def import_event_feedbacks :created_at => feedback["eval_time"] ) end - # update counts in batch - puts "[ ] updating feedbacks counters on events" if DEBUG - # FIXME speed problems: 26h - # Event.joins(:event_feedbacks).readonly(false).all.each {|e| e.recalculate_average_feedback!} - - # direct sqlite syntax: 10min? - ActiveRecord::Base.connection.execute "UPDATE events SET average_feedback=( - SELECT sum(rating)/count(rating) - FROM event_feedbacks WHERE events.id = event_feedbacks.event_id)" - - # other databases? - # UPDATE events - # SET average_feedback=(sum(rating)/count(rating)) - # FROM events - # INNER JOIN event_feedbacks - # ON id = event_id - # re-enable after_save callbacks - Event.set_callback(:save, :after, :update_conflicts) - EventFeedback.set_callback(:save, :after, :update_average) + # update in batch + puts "[ ] updating feedback average on events" if DEBUG + update_event_average("event_feedbacks", "average_feedback") + enable_event_callbacks(EventFeedback) end def import_event_attachments @@ -455,7 +470,8 @@ def interval_to_minutes(interval) def start_time(day, interval) return nil unless day and interval - Time.parse(day + " " + interval) + t = Time.parse(day + " " + interval) + Time.zone.utc_to_local(t).in_time_zone end def image_to_file(image, id_column) @@ -522,4 +538,35 @@ def save_mappings(name) def mappings_file(name) Rails.root.join('tmp', "#{name}_mappings.yml").to_s end + + def update_event_average(table, field) + # FIXME speed problems: 26h + # Event.joins(:event_feedbacks).readonly(false).all.each {|e| e.recalculate_average_feedback!} + + # direct sqlite syntax: 10min? + ActiveRecord::Base.connection.execute "UPDATE events SET #{field}=( + SELECT sum(rating)/count(rating) + FROM #{table} WHERE events.id = #{table}.event_id)" + + # other databases? + # UPDATE events + # SET average_feedback=(sum(rating)/count(rating)) + # FROM events + # INNER JOIN event_feedbacks + # ON id = event_id + end + + def disable_event_callback(average) + # we wan't to update this in batch after all the inserts + Event.skip_callback(:save, :after, :update_conflicts) + average.skip_callback(:save, :after, :update_average) + # TODO disable counter_cache? + end + + def enable_event_callbacks(average) + # re-enable after_save callbacks + Event.set_callback(:save, :after, :update_conflicts) + average.set_callback(:save, :after, :update_average) + end + end