Data exports

Gemfile

@@ -7,6 +7,8 @@ gem 'puma', '~> 4.3'
 
 gem 'panoptes-client'
 
+gem 'azure-storage-blob'
+
 # jsonapi.rb is a bundle that incorporates fast_jsonapi (serialization),
 # ransack (filtration), and some RSpec matchers along with some
 # boilerplate for pagination and error handling

Gemfile.lock

@@ -56,6 +56,17 @@ GEM
       minitest (~> 5.1)
       tzinfo (~> 1.1)
       zeitwerk (~> 2.2)
+    azure-core (0.1.15)
+      faraday (~> 0.9)
+      faraday_middleware (~> 0.10)
+      nokogiri (~> 1.6)
+    azure-storage-blob (1.1.0)
+      azure-core (~> 0.1.13)
+      azure-storage-common (~> 1.0)
+      nokogiri (~> 1.6, >= 1.6.8)
+    azure-storage-common (1.1.0)
+      azure-core (~> 0.1.13)
+      nokogiri (~> 1.6, >= 1.6.8)
     bootsnap (1.4.5)
       msgpack (~> 1.0)
     builder (3.2.3)
@@ -245,6 +256,7 @@ PLATFORMS
   ruby
 
 DEPENDENCIES
+  azure-storage-blob
   bootsnap (>= 1.4.2)
   coveralls
   factory_bot_rails

app/services/azure_blob_storage.rb

@@ -0,0 +1,41 @@
+require 'azure/storage/blob'
+
+class AzureBlobStorage
+  def initialize
+    # login
+    @blob_client = Azure::Storage::Blob::BlobService.create(
+      storage_account_name: Rails.application.credentials.storage_account_name,
+      storage_access_key: Rails.application.credentials.storage_access_key
+    )
+
+    # this is equivalent to S3 bucket, where have we stored bucket names
+    # in the past? I dont wanna keep it here, hardcoded.
+    # maybe put in the rails credentials file...? not sure.
+    @container_name = 'data-exports'
+  end
+
+  def put_file(blob_path, file)
+    content = ::File.open(file, 'rb') { |file| file.read }
+    @blob_client.create_block_blob(@container_name, blob_path, content)
+  end
+
+  # receive a list of hashes formatted as
+  # { blob_path: <path_to_blob>, file: <file_name> }
+  def put_files_multiple(file_list)
+    file_list.each do |f|
+      put_file(f[:blob_path], f[:file])
+    end
+  end
+
+  def get_file(path)
+    # to do
+  end
+
+  def delete_file
+    # to do
+  end
+
+  def get_files(prefix)
+    # to do
+  end
+end

app/services/data_exports.rb

@@ -0,0 +1,94 @@
+require 'fileutils'
+require 'securerandom'
+
+module DataExports
+
+  class DataExporter
+    def export_transcription(transcription_id)
+      file_generator = TranscriptionFileGenerator.new(transcription_id)
+      file_list = file_generator.generate_transcription_files
+
+      azure = AzureBlobStorage.new
+      azure.put_files_multiple(file_list)
+
+      # we are done with the files, delete the temp directory and its contents
+      file_generator.delete_temp_directory
+    end
+  end
+
+  class TranscriptionFileGenerator
+    def initialize(transcription_id)
+      @transcription = Transcription.find(transcription_id)
+      @directory_path = File.expand_path("~/transcription_files_temp/t#{transcription_id}_#{SecureRandom.uuid}")
+
+      FileUtils.mkdir_p @directory_path
+    end
+
+    def generate_transcription_files
+      file_list = []
+      blob_directory = generate_blob_directory(@transcription)
+
+      # raw data file
+      file_path = write_raw_data_to_file(@directory_path)
+      blob_path = File.join(blob_directory, "raw_data_#{@transcription.id}.json")
+      file_list.append({ :blob_path => blob_path, :file => file_path})
+
+      # consensus text file
+      file_path = write_consensus_text_to_file(@directory_path)
+      blob_path = File.join(blob_directory, "consensus_text_#{@transcription.id}.txt")
+      file_list.append({ :blob_path => blob_path, :file => file_path})
+    end
+
+    def delete_temp_directory
+      FileUtils.remove_dir(@directory_path)
+    end
+
+    private
+
+    # creates raw data file
+    # returns location of the file
+    def write_raw_data_to_file(directory_path)
+      file_path = File.join(directory_path, "raw_data_#{@transcription.id}.json")
+
+      File.open(file_path, 'w') { |f|
+        f.puts @transcription.text
+      }
+      file_path
+    end
+
+    # creates raw data file
+    # returns location of the file
+    def write_consensus_text_to_file(directory_path)
+      file_path = File.join(directory_path, "consensus_text_#{@transcription.id}.txt")
+
+      File.open(file_path, 'w') { |f|
+        f.puts consensus_text
+      }
+
+      file_path
+    end
+
+    def consensus_text
+      consensus_text = ""
+      @transcription.text.each do |key, value|
+        # if we find a frame, iterate through the lines of the frame
+        if /^frame/.match(key)
+          value.each do |line|
+            puts line["consensus_text"]
+            consensus_text.concat "#{line["consensus_text"]} "
+          end
+        end
+      end
+
+      consensus_text
+    end
+
+    def generate_blob_directory(transcription)
+      workflow_id = transcription.workflow_id
+      project_id = Workflow.find(workflow_id).project_id
+
+      "approved-transcriptions/#{project_id}/#{workflow_id}/#{transcription.group_id}/#{transcription.id}"
+    end
+  end
+
+end

app/services/zip_file_generator.rb

@@ -0,0 +1,11 @@
+class ZipFileGenerator
+  # Initialize with the directory to zip and the location of the output archive.
+  def initialize(input_dir, output_file)
+    @input_dir = input_dir
+    @output_file = output_file
+  end
+
+  def write_to_file
+    # TO DO
+  end
+end

config/credentials/development.yml.enc

@@ -1 +1 @@
-BDFosYTQ4IYJUiWlYbHhn1pF7w615FMHDFWk0HdaLpIsMlnrRF9j8Jz7M1T8f6KyLqMA4IalPb8ZMNDsgjYPPUgVsvPcxnO27fU3T45iR0SFdXu2yZj8J/KCabg/TZC7zK5tz0AwKQKDJbZq4QgRAZJ0GjcLCuRT22EO9bQjj00xhEvKRPESIvLIkfbc9+xVpafrp7ugpuIU1QEe8w40VNSgGuVdcKSysosskUAx5Me7AluxWt2AmmFhfK9D5cWYi5hfSHsg8FadDTYbpuVdIU3I4sTUBGvb336Tnp83VHZSsQ61DN8IVF9yivBPGq4xmAayp7MIBLgg0JQLQ/jCnsDOGdTMnAtnV5Bn+p5WmGFyexx8Obis5frc5LP7dfwsI3PE46HNbifGsxe5O3/VvgRpJzrUrWcmU9njYmbQ5Lor48L5Ffytfw8xgc91Dl/xhJvAVl7UHduAU7YfmLbLqYixtTOgtJ1s3I8oM0W6B3xf+X1Xh3LlPg839Nu5kGEPUIrLeXJfFOCXxoyichzei8agAnYY7n8hm2dYnQQ8fUFuPy0qPkjMzI1XYOKXLD+cHNXF0bmHuKmSutg2IgfuTz2TWIbZvQz05RAKPb111MFqJTYRYDcRaxI2kMbEpTogKWSXV2Co8A==--8m86VjL9ErzM0pYo--lDHySsl4pCt2k5vxlc8nTQ==
+xLnPctk/D5tdYwgubgdaXFZynrG9FOSowt/3aR4AdC9Up6Kf/xeNX4+J3829RxmWdj3HnL90vK9QEP+LZq0U8fRHGfB3Om5G2KPeYQHxCcwVh4zKmi2BzcbWpt0+3vmhhmjU/iSwOH1I9y8Ary0tKUzbIATIB+wb+f/dEorWkxe7YU9YT3+Br3FCqTvVnxjuBVy+NxFTFSeIGWTMycTUXlOUmrE2cijIKx5AGXbN2h1x8BOTJehtzGrqQFi/9gMUrYMWOmqHPr3WvWQ8Kb/SqQRWiFxYWYIbNQK5x3KUpISJSI03NvA1QbMxXQl8PKDOFOfp215fyHg4bxT9zSe2o6P9sABx90CEnt1HYYgv+Qgqe1j+SA/qjx6seQ58y2UQpO1xiVhff4BLFJI+Gic4tN/aPtR7z/zxSEpHRcKDKXyVXKXSsxWZxG7/NbIF/9QBdpf/RD9qJJN8v5s7tWGGW0x4vCTkB+121HG2u+PSbDPYkKdhUY9XYCjm1ULx9eYDwrYLVZ7sLd/Mta3I1iC6gOZC80jtFpM8CSOs36Qhuhv1lkZCaWDlVViHp/RGQjS8hLQ39JKscrIj4VIxgqGsfNdBGAF8NpW+uoFircuvfMXaA8Ed0UwE2nJ4pQkPhG9jBiIfzbjb/mcyqwEP6SuHbLO5P+6mPM+NXOtv+DqQ7meuieVVf4Cgs88VDOp2UqlB3T0ECF1/CjEL5ToKIANuBA5S/ijqlXzgTR5qhthhmepSrMes9SvlICzPUsOqzGwgo9DWZBSnGRMlBqdR9oj5ZX5JkjPPzyE+Dmpo3gfSNJhekv7yIghgzPG4R34XllznewFe6k5U0W4SCjMHWjTJ+g0=--l7vzkTuEoY0PFhLC--ox5cxai5P9nHkH2B6OBwRw==

config/credentials/staging.yml.enc

@@ -1 +1 @@
-dbVsO7tt8C7NtzVaASevcKoqzdud39C0UwQcPbAvovNBODYd6sSJOpQWBs+DcA8ASX6ZMoRmHEmYmXb8ShQOpF1kDNXdylETILmVts5FBs4xmKguqWsYlNNYqqGagrw1NoZDWUt6TQotM6Rol2B1rQsdVdRbjGUr9NdnKy7liLpZolF9gBIRByHUZXjI/waOzkmJE62EhgDaqWCUdmlCNqk+zw/q0nw83ZxCjwngOKAObEcrGwsrxdIbhHbIWeQAUprZuWdIcvVtajrjul22zyWE31dqCD7L58jFxKnVMiv9a4N28aINAT4D5gnDl7gtmWMZEhSI0Oof0a8rdyCnaNQSjOGCNyPnvF175L+SVsnyH/jrEQy/z4jB0wOH8Hol9sh30k0PlCuNgcP8yzkpmfvuXeklFDueImuDBqJcnpDnYgN4GyhuaZCJ3eVC4s8M8NNcLfEtEkbXi5pJ3jlBLWdMM2nxZjotwhEbvhtAE8dOZ8jhayO15ScH05XVx50A/Ca/cNL4Mcfy/NzbwSr+yXL1HznliC5oxSCe37ENVBe9UifxAsKhbBYTJSrcUew66jZRv81rSKFpAOqSe0PolipJT/VDgHjtusXF7TUw1INgRIdVnNBcas8KAco2CtQnBtr6XGB8uPz+m+yJ6Q3Yp8D1e5Q2h4ygQLVd/pXvYwFNELnQQPgY1Xv5bF5q4NFy9zZfQMhZnjnW2ylvquY95l3ObtdVDqQ05n3rO8bm//IZJPBqDVo62b3UnRa5OUM0+Nb7H772xR5MAgUWL5u9zThA9idcBLpypkwM5++TGWATgDwuTOJB9WNq+JTHYRVo1LL++8m5sMfSQkn7WH2Oi5svhRyFxoy/8gvINbi3FxfQf0Jq7pP0zcKLyYW96JeWcYM/sIgMC78Btl58LoSuIxktiO8EQevXc3zGz38OIGhKeEzf--IRKoGrapAb6uBc5o--jPNtDC+nLdKsVYTodvFa9g==
+b5Viyt9F/180us5F0aH4lWUYleVhmOeiNoEcPCKJuOidIIAwPbVecvRyFL7r6nDFF/4r8nV/dJ2vO9OI5Idbkz9mXHAw6w+Cd4w5MdbqLzyREn8VU5a/XXY4uHKTjyVlLKb+E7RzlJy3YeXAl1rrjGmvsFNcMVtYzCZtV2CwZdd5BHrCisZygXMz7EnbYPRd3GSQRMYC24utj25nXGQwkv5Bkv6PfpiF1rAZ5bwDqdMCJTV4HQhn+5SXxtyeJZZ7SUE3gmF5SEkYKq+akUuOTxJZRLZylIG0OFaiG/dWHX5U5oCU7MiEt1gm3Z2/iDbBi7veziskzyDyYZrp7iIQIHVG2LGDHXU2lqcPg1ayYiaVMfB4xfaPLbTIX7sIt6ysGKOw7KySNFaak8ZAoUXlkAeCqlj3noru69th6ph375kn2F+R2/6Dxymfk3l7+NSPCCFrMLBM8KYQOuJ4G38/Gj+l+s2l2+WkyZ5J/Wj0YENniAHwRk4TUSII76gV6dCr90s8Fd2hRkAGvKFyn9ztco+/5QO5e95JsyeJU7fPtnYzei3agSdrf6IC15l86p9wnuVL/G/w6fECP00M2GobED/COnObU/4H8u0EfZtysWZ0bUpnvBviJMtY1cSy+vCegrM1srSW/6SiBuJ6jovyYE8UhzI0ebgF/FWH45xXiGSQjRkj/JX67vqDEY/X07D24f6OgaTfv+NSWfA6b82gANdvXZAfyxhCIxV9XbEoF+jWw1m48iNO57KbbcuvWQWbcW3z5FgJT9jJ9pKNZ4T6KJL8y3njkG3ffud0qkGReKpvhsRrCVMnB4kBSlNkVuBPSd22boAH2USW/+jumJE0Tdjxo/gZZxljswmL6zqo8yWEJsTqhir0VtOzhSzPVTjX3guEUW7velTqi4JclqdWgziKYyFIZJu+BjWYdFuGVA6dL9oApnzhIJj35INQ1ICK+lQbGQs6p/RsI3wCqBQHPg23AHOIgI6G2Bz1tidEyVmIyUaozPzvSOIdaxaWoGDQzDKidY0ZPXQhnJlYXaXGZCJ1+rodgnUYbvgyxRyJK0j+CYRJIkzrR7mlKU6BW0KWSpfNbe/YqpIw53oILpyi9lXUXhHUCsaefENGkVFqFGsc55Yi0hkElu8doxW8xg==--s7gLpzyjZZ/Uj1qh--DwPwGF7dD3xgAfT71g5+HQ==

spec/services/data_exports_spec.rb

@@ -0,0 +1,9 @@
+RSpec.describe TranscriptionFileGenerator, type: :service do
+  describe '#generate_transcription_files' do
+    let(:transcription){ create :transcription }
+
+    it 'does something' do
+
+    end
+  end
+end