From 5449b8f7e987b25e648616f44832757b603ff725 Mon Sep 17 00:00:00 2001 From: Daniel Kukula Date: Sat, 5 Jan 2019 14:24:33 +0000 Subject: [PATCH 1/6] rake task for dumping, restoring and anonymizing user data --- backend/lib/tasks/db.rake | 57 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100644 backend/lib/tasks/db.rake diff --git a/backend/lib/tasks/db.rake b/backend/lib/tasks/db.rake new file mode 100644 index 000000000..a85d0e5bd --- /dev/null +++ b/backend/lib/tasks/db.rake @@ -0,0 +1,57 @@ +namespace :db do + + desc 'replace user sensitive data with placeholders' + task anonymize_user: :environment do + if ENV['RAILS_ENV'] == 'production' + puts 'You do not want to anonymize production data' + else + puts 'Anonymizing user data' + + User.all.each do |user| + user.encrypted_password = user.encrypted_password.truncate(8) + user.latitude = 50 + 0.001 * user.id + user.longitude = 10 - 0.001 * user.id + user.city = "city#{user.id}" + + user.email = "user#{user.id}@rundfunk.com" + user.save! + end + end + end + + desc 'Dumps the database to db/APP_NAME.dump' + task dump: :environment do + cmd = nil + app, host, db, user = with_config + cmd = "pg_dump --host #{host} #{user_present?(user)} --verbose --clean --no-owner --no-acl --format=c #{db} > #{Rails.root}/db/#{app}.dump" + puts cmd + exec cmd + end + + desc 'Restores the database dump at db/APP_NAME.dump.' + task restore: :environment do + cmd = nil + app, host, db, user = with_config + cmd = "pg_restore --verbose --host #{host} #{user_present?(user)} --clean --no-owner --no-acl --dbname #{db} #{Rails.root}/db/#{app}.dump" + Rake::Task['db:drop'].invoke + Rake::Task['db:create'].invoke + Rake::Task['db:migrate'].invoke + puts cmd + exec cmd + end + + private + + def user_present?(user) + user ? '--username ' + user : nil + end + + def with_config + [ + Rails.application.class.parent_name.underscore, + ActiveRecord::Base.connection_config[:host], + ActiveRecord::Base.connection_config[:database], + ActiveRecord::Base.connection_config[:username] + ] + end +end From e6ca634661ee6ecc863a527446fc6dbdcae54175 Mon Sep 17 00:00:00 2001 From: Daniel Kukula Date: Sat, 5 Jan 2019 22:09:17 +0000 Subject: [PATCH 2/6] corrected CI offenses --- backend/lib/tasks/db.rake | 3 --- 1 file changed, 3 deletions(-) diff --git a/backend/lib/tasks/db.rake b/backend/lib/tasks/db.rake index a85d0e5bd..979f80fd2 100644 --- a/backend/lib/tasks/db.rake +++ b/backend/lib/tasks/db.rake @@ -1,5 +1,4 @@ namespace :db do - desc 'replace user sensitive data with placeholders' task anonymize_user: :environment do if ENV['RAILS_ENV'] == 'production' @@ -21,7 +20,6 @@ namespace :db do desc 'Dumps the database to db/APP_NAME.dump' task dump: :environment do - cmd = nil app, host, db, user = with_config cmd = "pg_dump --host #{host} #{user_present?(user)} --verbose --clean --no-owner --no-acl --format=c #{db} > #{Rails.root}/db/#{app}.dump" puts cmd @@ -30,7 +28,6 @@ namespace :db do desc 'Restores the database dump at db/APP_NAME.dump.' task restore: :environment do - cmd = nil app, host, db, user = with_config cmd = "pg_restore --verbose --host #{host} #{user_present?(user)} --clean --no-owner --no-acl --dbname #{db} #{Rails.root}/db/#{app}.dump" Rake::Task['db:drop'].invoke From 47a14814b711e1fdc75f20053d625bf321bb1520 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Wed, 9 Jan 2019 19:02:53 +0000 Subject: [PATCH 3/6] Update backend/lib/tasks/db.rake Co-Authored-By: dkuku --- backend/lib/tasks/db.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/tasks/db.rake b/backend/lib/tasks/db.rake index 979f80fd2..6430cd057 100644 --- a/backend/lib/tasks/db.rake +++ b/backend/lib/tasks/db.rake @@ -6,7 +6,7 @@ namespace :db do else puts 'Anonymizing user data' - User.all.each do |user| + User.find_each do |user| user.encrypted_password = user.encrypted_password.truncate(8) user.latitude = 50 + 0.001 * user.id user.longitude = 10 - 0.001 * user.id From baa8e0c3252459533fa2e24a4d9f739bae74d4e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Robert=20Sch=C3=A4fer?= Date: Wed, 9 Jan 2019 19:03:04 +0000 Subject: [PATCH 4/6] Update backend/lib/tasks/db.rake Co-Authored-By: dkuku --- backend/lib/tasks/db.rake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/lib/tasks/db.rake b/backend/lib/tasks/db.rake index 6430cd057..4eda06aea 100644 --- a/backend/lib/tasks/db.rake +++ b/backend/lib/tasks/db.rake @@ -12,7 +12,7 @@ namespace :db do user.longitude = 10 - 0.001 * user.id user.city = "city#{user.id}" - user.email = "user#{user.id}@rundfunk.com" + user.email = "user#{user.id}@example.org" user.save! end end From d7a700bda1f9da7350f70f62a38ee4ddb9006811 Mon Sep 17 00:00:00 2001 From: Daniel Kukula Date: Thu, 10 Jan 2019 03:19:34 +0000 Subject: [PATCH 5/6] Anonymize Broadcast.creator_id --- backend/lib/tasks/db.rake | 22 +++++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/backend/lib/tasks/db.rake b/backend/lib/tasks/db.rake index 979f80fd2..18107060d 100644 --- a/backend/lib/tasks/db.rake +++ b/backend/lib/tasks/db.rake @@ -6,15 +6,24 @@ namespace :db do else puts 'Anonymizing user data' - User.all.each do |user| + @user_ids = User.ids.shuffle + Broadcast.find_each do |broadcast| + broadcast.creator_id = @user_ids.sample + broadcast.save! + end + User.find_each do |user| + new_id = @user_ids.pop user.encrypted_password = user.encrypted_password.truncate(8) - user.latitude = 50 + 0.001 * user.id - user.longitude = 10 - 0.001 * user.id - user.city = "city#{user.id}" + user.auth0_uid = user.auth0_uid.truncate(15) + new_id.to_s if user.auth0_uid + user.latitude = 50 + 0.001 * new_id + user.longitude = 10 - 0.001 * new_id + user.city = "city#{new_id}" + user.postal_code = 10000 + new_id - user.email = "user#{user.id}@rundfunk.com" + user.email = "user#{new_id}@example.com" user.save! end + Rake::Task['db:dump'].invoke end end @@ -33,6 +42,9 @@ namespace :db do Rake::Task['db:drop'].invoke Rake::Task['db:create'].invoke Rake::Task['db:migrate'].invoke + #TODO does not work + Rails.env = 'development' + Rake::Task['db:environment:set'].invoke puts cmd exec cmd end From 0d68f00b39a2b8229b8d2a5df5d9040b9d96e18f Mon Sep 17 00:00:00 2001 From: Daniel Kukula Date: Thu, 10 Jan 2019 03:20:54 +0000 Subject: [PATCH 6/6] Remove unused code --- backend/lib/tasks/db.rake | 4 ---- 1 file changed, 4 deletions(-) diff --git a/backend/lib/tasks/db.rake b/backend/lib/tasks/db.rake index 18107060d..a45006ef2 100644 --- a/backend/lib/tasks/db.rake +++ b/backend/lib/tasks/db.rake @@ -5,7 +5,6 @@ namespace :db do puts 'You do not want to anonymize production data' else puts 'Anonymizing user data' - @user_ids = User.ids.shuffle Broadcast.find_each do |broadcast| broadcast.creator_id = @user_ids.sample @@ -42,9 +41,6 @@ namespace :db do Rake::Task['db:drop'].invoke Rake::Task['db:create'].invoke Rake::Task['db:migrate'].invoke - #TODO does not work - Rails.env = 'development' - Rake::Task['db:environment:set'].invoke puts cmd exec cmd end