Refactor Background Jobs

Dockerfile

@@ -6,7 +6,7 @@ RUN apt-get update -qq --fix-missing
 RUN apt-get install -y curl
 
 # Get node
-RUN curl -sL https://deb.nodesource.com/setup_8.x | bash -
+RUN curl -sL https://deb.nodesource.com/setup_10.x | bash -
 RUN apt-get update && apt-get install -y nodejs apt-transport-https
 
 RUN apt-get install -y xvfb nano build-essential libpq-dev wget postgresql-client postgresql-contrib

app/assets/stylesheets/extensions/compilation.scss

@@ -36,7 +36,7 @@
   top: 0;
   font-size: smaller;
   margin-bottom: 1.5em;
-  .compilation-complete {
+  .compilation-pending {
     border-radius: 5px;
     border: 1px solid $notice_primary_green;
     background-color: $notice_secondary_green;
@@ -52,6 +52,10 @@
     border: 1px solid $notice_primary_red;
     background-color: $notice_secondary_red;
   }
+  .compilation-complete {
+    border: none;
+    background-color: white;
+  }
 }
 
 .compilation-notice {

app/controllers/extensions_controller.rb

@@ -47,7 +47,6 @@ def new
       @repo_names = Marshal.load(@repo_names)
     else
       FetchAccessibleReposWorker.perform_async(current_user.id)
-      # puts '******* FetchAccessibleReposWorker *********'
     end
 
     @extension = Extension.new
@@ -60,15 +59,23 @@ def new
   #
   def create
     eparams = params.require(:extension).permit(:name, :description, :github_url, :github_url_short, :tmp_source_file, :tag_tokens, :version, compatible_platforms: [])
-    create_extension = CreateExtension.new(eparams, current_user)
-    @extension = create_extension.process!
 
-    if @extension.errors.none?
-      redirect_to owner_scoped_extension_url(@extension), notice: t("extension.created")
+    candidate = Extension.new(eparams)
+
+    if candidate.valid?
+      create_context = CreateExtension.call(params: eparams, candidate: candidate, user: current_user)
+
+      if create_context.success?
+        @extension = create_context.extension
+        redirect_to owner_scoped_extension_url(@extension), notice: t("extension.created")
+        return 
+      end
     else
-      @repo_names = current_user.octokit.repos.map { |r| r.to_h.slice(:full_name, :name, :description) } rescue []
-      render :new
+      flash[:alert] = candidate.errors.full_messages.join("; ")
     end
+    @extension = Extension.new
+    @repo_names = current_user.octokit.repos.map { |r| r.to_h.slice(:full_name, :name, :description) } rescue []
+      render :new
   end
 
   #
@@ -169,14 +176,14 @@ def update
     @extension.update_attributes(extension_edit_params)
 
     key = if extension_edit_params.key?(:up_for_adoption)
-            if extension_edit_params[:up_for_adoption] == 'true'
-              'adoption.up'
-            else
-              'adoption.down'
-            end
-          else
-            'extension.updated'
-          end
+      if extension_edit_params[:up_for_adoption] == 'true'
+        'adoption.up'
+      else
+        'adoption.down'
+      end
+    else
+      'extension.updated'
+    end
 
     redirect_to owner_scoped_extension_url(@extension), notice: t(key, name: @extension.name)
   end
@@ -328,19 +335,33 @@ def report
     redirect_to owner_scoped_extension_url(@extension), notice: t("extension.reported", extension: @extension.name)
   end
 
+  #
+  # GET /extensions/:extension/sync_repo
+  #
+  # Recompiles the asset
+  #
   def sync_repo
-    extension = Extension.with_owner_and_lowercase_name(owner_name: params[:username], lowercase_name: params[:id])
-    authorize! extension
-    CompileExtension.call(extension: extension)
-    redirect_to owner_scoped_extension_url(@extension), notice: t("extension.syncing_in_progress")
+    @extension = Extension.with_owner_and_lowercase_name(owner_name: params[:username], lowercase_name: params[:id])
+    authorize! @extension
+    compile_context = CompileExtension.call(extension: @extension)
+    if compile_context.success?
+      flash[:notice] = t("extension.syncing_in_progress")
+    else
+      flash[:error] = t("extension.failed_to_compile", extension: @extension.name)
+    end
+    redirect_to owner_scoped_extension_url(@extension)
   end
 
+  #
+  # GET /extensions/:extension/sync_status
+  #
+  # Returns the status of a recompile
+  #
   def sync_status
     @extension = Extension.find_by(id: params[:id])
     redis_pool.with do |redis|
-      @job_ids = JSON.parse( redis.get("compile.extension;#{params[:id]};status") || "{}" )
+      @status = JSON.parse( redis.get("compile.extension;#{params[:id]};status") || "{}" )
     end
-
     respond_to do |format|
       format.js
     end
@@ -379,9 +400,9 @@ def webhook
           CollectExtensionMetadataWorker.perform_async(@extension.id, [])
         end
       when "watch"
-        ExtractExtensionStargazersWorker.perform_async(@extension.id)
+        ExtractExtensionStargazers.call(@extension)
       when "member"
-        ExtractExtensionCollaboratorsWorker.perform_async(@extension.id)
+        ExtractExtensionCollaborators.call(@extension)
       else
         Rails.logger.info '*** Github Error: unidentified event type'
     end

app/helpers/extensions_helper.rb

@@ -187,6 +187,14 @@ def to_asset_definition(version, format: "yaml")
     resp
   end
 
+  #
+  # Given extentsion returns compilation errors
+  #
+  # @param extension [Extension] 
+  # @param version [ExtensionVersion] optional
+  #
+  # @return [String] compilation errors
+  #
   def compilation_errors(extension, version=nil)
     errors = []
     if extension.compilation_error.present?
@@ -200,53 +208,32 @@ def compilation_errors(extension, version=nil)
     errors
   end
 
-  def compilation_status(extension, job_ids)
-    job_count = @job_ids.length
-    pending, retrying, complete, failed = 0.0, 0.0, 0.0, 0.0
-    jobs_failed, jobs_retrying = [], []
-
-    @job_ids.each do |job, key|
+  def compilation_status(extension, status)
+    worker_status = Sidekiq::Status::status( status['job_id'] ) 
+    html = ''
+    klass = ''
+    case worker_status
+      when :queued, :working
+        html += content_tag('div', raw("<b>Compiling Extension:</b> #{status['task']}"), class: 'compilation-notice')
+        klass = 'compilation-pending'
 
-      status = Sidekiq::Status::status(key)
+      when :retrying
+        html += content_tag("div", raw("<b>Retrying Compilations:</b> #{status['worker']}"), class: 'compilation-notice')
+        klass = 'compilation-retry'
 
-      case status
-        when :queued, :working
-          pending += 1
-        when :retrying
-          retrying += 1
-          jobs_retrying << job
-        when :complete
-          complete += 1
-          # puts status
-        when :failed, :interrupted
-          failed += 1
-          jobs_failed << job
-        else
-      end
-    end
-
-    percent_complete = ((complete / job_count) * 100).round(2)
+      when :complete
+        redis_pool.with do |redis|
+          redis.del("compile.extension;#{extension.id};status")
+        end
+        klass = 'compilation-complete'
 
-    html = ''
-    klass = 'compilation-complete'
-    if failed > 0
-      html += content_tag("div", raw("<b>Compilations failed:</b> #{jobs_failed.join(', ')}"), class: 'compilation-notice')
-      klass = 'compilation-failed'
-    end
-    if retrying > 0
-      html += content_tag("div", raw("<b>Retrying Compilations:</b> #{jobs_retrying.join(', ')}"), class: 'compilation-notice')
-       klass = 'compilation-retry'
-    end
-    html += content_tag('div', raw("<b>Compiling Extension % #{percent_complete} Complete</b>"), class: 'compilation-notice')
+      when :failed, :interrupted
+        html += content_tag("div", raw("<b>Compilations failed:</b> #{status['worker']}"), class: 'compilation-notice')
+        klass = 'compilation-failed'
 
-    if percent_complete == 100
-      redis_pool.with do |redis|
-        redis.del("compile.extension;#{extension.id};status")
-      end
     end
-
     content_tag('div', raw(html), class: klass )
-
+  
   end
 
   #

app/interactors/compile_extension.rb

@@ -5,39 +5,38 @@
 class CompileExtension
   include Interactor
 
-  # The required context attributes:
   delegate :extension, to: :context
 
   def call
+
+    extension.update_column(:compilation_error, '')
+
+    # test credentials before going to background job
+    begin
+      extension.octokit.user
+    rescue Octokit::Unauthorized 
+      message = "Octokit Error: Credentials are bad on this asset."
+      extension.update_column(:compilation_error, message)
+      context.fail!(error: message)
+    end  
+
+    CompileExtensionStatusClear.call(extension: extension)
+
     if extension.hosted?
-      compile_hosted_extension(extension)
-    else
-      CompileExtensionStatusClear.call(extension: extension)
-
       CompileExtensionStatus.call(
-        extension: extension, 
-        worker: 'ExtractExtensionParentWorker', 
-        job_id: ExtractExtensionParentWorker.perform_async(extension.id)
+        extension: extension,
+        task: 'Set Up Hosted Extension Compilation',
+        worker: 'SetupCompileHostedExtension',
+        job_id: SetupCompileHostedExtensionWorker.perform_async(extension.id)
       )
-      
+    else
       CompileExtensionStatus.call(
-        extension: extension, 
-        worker: 'SyncExtensionRepoWorker', 
-        job_id: SyncExtensionRepoWorker.perform_async(extension.id) 
+        extension: extension,
+        task: 'Set Up Github Extension Compilation',
+        worker: 'SetupCompileGithubExtension',
+        job_id: SetupCompileGithubExtensionWorker.perform_async(extension.id)
       )
     end
   end
 
-  private
-
-  def redis_pool
-    REDIS_POOL
-  end
-
-  def compile_hosted_extension(extension)
-    extension.extension_versions.each do |version|
-      next unless version.source_file.attached?
-      version.source_file.blob.analyze_later
-    end
-  end
 end

app/interactors/compile_extension_status.rb

@@ -2,15 +2,21 @@ class CompileExtensionStatus
   include Interactor
 
   # The required context attributes:
-  delegate :extension, to: :context
-  delegate :worker, to: :context 
-  delegate :job_id, to: :context
+  delegate :extension,  to: :context
+  delegate :task,       to: :context
+  delegate :worker,     to: :context 
+  delegate :job_id,     to: :context
+
 
   def call
   	redis_pool.with do |redis|
   		status = redis.get("compile.extension;#{extension.id};status")
 	  	status = status.present? ? JSON.parse( status ) : {}
-	    status[worker] = job_id
+      status['task'] = task.present? ? task : ''
+      if job_id.present?
+        status['worker'] = worker
+        status['job_id'] = job_id
+      end 
 	    redis.set("compile.extension;#{extension.id};status", status.to_json)
 	  end
   end