From f7e1ae187ce4a052e4bcce70c155654e37bb932b Mon Sep 17 00:00:00 2001
From: Josh Larsen <2565382+joshlarsen@users.noreply.github.com>
Date: Fri, 28 May 2021 10:18:47 -0400
Subject: [PATCH] better handling of malformed JSON import files. #176

---
 .github/ISSUE_TEMPLATE/bug_report.md             |  3 +++
 docker/app/jobs/lib/loader/asset/asset_router.rb |  8 ++++++++
 docker/app/jobs/lib/loader/file_loader.rb        | 11 ++++++++---
 docker/docker-compose.yml                        |  2 +-
 4 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md
index 79412b3..964d21e 100644
--- a/.github/ISSUE_TEMPLATE/bug_report.md
+++ b/.github/ISSUE_TEMPLATE/bug_report.md
@@ -10,6 +10,9 @@ assignees: ''
 **Describe the bug**
 A clear and concise description of what the bug is.
 
+**OpenCSPM version**
+What version of the OpenCSPM Docker image are you using? (e.g. 0.1.22)
+
 **To Reproduce**
 Steps to reproduce the behavior:
 1. Go to '...'
diff --git a/docker/app/jobs/lib/loader/asset/asset_router.rb b/docker/app/jobs/lib/loader/asset/asset_router.rb
index 70e0bc7..d66d201 100644
--- a/docker/app/jobs/lib/loader/asset/asset_router.rb
+++ b/docker/app/jobs/lib/loader/asset/asset_router.rb
@@ -17,6 +17,8 @@
 
 # Comment
 class AssetRouter
+  INVALID_JSON = 'Expected NDJSON import file, found JSON import file.'
+
   def initialize(asset, import_id, db)
     @asset = asset
     @import_id = import_id
@@ -25,6 +27,12 @@ def initialize(asset, import_id, db)
   end
 
   def route
+    # raise exception if not NDJSON
+    if @asset.is_a?(Array)
+      puts "[asset_router] Error: #{INVALID_JSON}"
+      raise INVALID_JSON
+    end
+
     # Load an AWS resource
     return aws_load if (%w[account service region resource] - @asset.keys).empty?
 
diff --git a/docker/app/jobs/lib/loader/file_loader.rb b/docker/app/jobs/lib/loader/file_loader.rb
index 507f28a..7e9116f 100644
--- a/docker/app/jobs/lib/loader/file_loader.rb
+++ b/docker/app/jobs/lib/loader/file_loader.rb
@@ -45,9 +45,14 @@ def loop_over_asset_lines(file_name)
       file.each_slice(batch_size) do |lines|
         line_count += lines.length
         Parallel.each(lines, in_processes: 8) do |line|
-          asset_json = FastJsonparser.parse(line, symbolize_keys: false)
-          AssetRouter.new(asset_json, @import_id, @db)
-          line_count += 1
+          begin
+            asset_json = FastJsonparser.parse(line, symbolize_keys: false)
+            AssetRouter.new(asset_json, @import_id, @db)
+            line_count += 1
+          rescue FastJsonparser::ParseError => e
+            puts "[file_loader] Error: JSON can't be parsed. Ensure you are loading NDJSON files."
+            raise e
+          end
         end
       end
     end
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 6e7cc24..b603133 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -6,7 +6,7 @@ x-app: &app
     RAILS_ENV: ${RAILS_ENV:-production}
     RAILS_SERVE_STATIC_FILES: 1
     RAILS_LOG_TO_STDOUT: 1
-  image: gcr.io/opencspm/engine:0.1.22
+  image: gcr.io/opencspm/engine:0.1.23
   tmpfs:
     - /tmp