Merge pull request #94 from SpiderLabs/new_api_update

Updated to use the new API from NVD

Gemfile

@@ -19,3 +19,5 @@ group :development, :test do
   gem 'rack-test', '~> 1.1.0'
   gem 'simplecov', '~> 0.17.0', :require => false
 end
+
+gem "tzinfo", "~> 2.0"

Gemfile.lock

@@ -83,6 +83,8 @@ GEM
       net-scp (>= 1.1.2)
       net-ssh (>= 2.8.0)
     tilt (2.0.9)
+    tzinfo (2.0.4)
+      concurrent-ruby (~> 1.0)
 
 PLATFORMS
   ruby
@@ -101,6 +103,7 @@ DEPENDENCIES
   simplecov (~> 0.17.0)
   sinatra (~> 2.0.7)
   sinatra-contrib (~> 2.0.7)
+  tzinfo (~> 2.0)
 
 BUNDLED WITH
-   1.16.1
+   1.17.3

README.md

@@ -64,30 +64,18 @@ curl --ssl -s https://raw.githubusercontent.com/SpiderLabs/cve_server/master/scr
 
     bundle install
 
-  3. Download the raw data from the National Vulnerability Database. The supported
-  NVD reports are [XML 2.0 (by default) and JSON 1.0 files](https://nvd.nist.gov/vuln/data-feeds).
-
-    ./bin/nvd_downloader
-
-    or
-
-    ./bin/nvd_downloader -f json
-
-  4. Configure your database.
+  3. Configure your database.
 
     vi config/database.yml
 
-  5. Create and populate the database for you environment.
-
-    RACK_ENV=development ./bin/seed
-
-    or
-
-    RACK_ENV=development ./bin/seed -f json
-
-  The `-f` flag with the `json` option will populate the database using the experimental JSON reports from NVD and it renames the `score` key to `base_score` in the `cvss` (v2) field, it also includes the `cvssv3` information and some changes for  the links in the `references` field.
+  4. Download, create and populate the database for your environment from the National Vulnerability Database via the NVD CVE/CPE API.
+     Note: The new API service is JSON only.    
+     [NVD API URL](https://services.nvd.nist.gov/rest/json/cves/1.0).
 
-  6. Start the server.
+    RACK_ENV=development ./bin/nvd_download_and_seed
+    ** The download may take hours to complete **
+
+  5. Start the server.
 
     RACK_ENV=development puma
 

bin/nvd_download_and_seed

@@ -0,0 +1,46 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__)
+require 'zlib'
+require 'json'
+require 'net/http'
+require 'tzinfo'
+require 'nvd_helper'
+require 'cve_server'
+require 'cve_server/nvd/json/reader'
+
+class NvdSeed
+  include NVDHelper
+
+  def download_data
+    if local_data_timestamp = last_data_timestamp
+      print "There are files in the nvd_data directory. Quitting.\n"
+      exit
+    else
+      files = download_pages
+      update_db(files)
+    end
+  end
+
+
+  def update_db(file_list)
+    CVEServer::Cve.drop_all
+    CVEServer::Cpe.drop_all
+    CVEServer::CpeWithVersion.drop_all
+
+    file_list.sort.each do |file|
+      puts "Uncompressing #{file}"
+      input = Zlib::GzipReader.open(file).read
+      @doc = CVEServer::NVD::JSON::Reader.new(input)
+      puts 'Exporting data into the CVE collection'
+      CVEServer::Cve.bulk_create(@doc.all_cve)
+    end
+    puts "Reducing the cpe list"
+    CVEServer::Cve.reduce_cpes
+    puts 'Creating CVE collection Index'
+    CVEServer::Cve.create_index(:cve)
+  end
+end
+
+nvd_seed = NvdSeed.new
+nvd_seed.download_data
+

bin/nvd_download_and_update

@@ -0,0 +1,38 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__)
+require 'json'
+require 'net/http'
+require 'tzinfo'
+require 'nvd_helper'
+require 'cve_server'
+require 'cve_server/nvd/json/reader'
+
+class NvdUpdater
+  include NVDHelper
+
+  def download_data
+    if local_data_timestamp = last_data_timestamp
+      files = download_pages(local_data_timestamp)
+      update_db(files)
+    else
+      print "There are no local files to aquire a timestamp from for updating.\n"
+    end
+  end
+
+
+  def update_db(file_list)
+    file_list.sort.each do |file|
+      puts "Uncompressing #{file}"
+      input = Zlib::GzipReader.open(file).read
+      @doc = CVEServer::NVD::JSON::Reader.new(input)
+      puts 'Exporting data into the CVE collection'
+      CVEServer::Cve.bulk_upsert(@doc.all_cve)
+    end
+    puts "Reducing the cpe list"
+    CVEServer::Cve.reduce_cpes
+  end
+end
+
+nvd_updater = NvdUpdater.new
+nvd_updater.download_data
+

bin/seed

@@ -2,27 +2,9 @@
 $LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__)
 require 'zlib'
 require 'cve_server'
-require 'cve_server/nvd/reader'
 require 'cve_server/nvd/json/reader'
-require 'optparse'
 
-usage = 'Usage: seed [options]'
-options = { file_ext: 'json' }
-OptionParser.new do |opts|
-  opts.banner = usage
-
-  # Mandatory argument.
-  opts.on("-f", "--file_extension json or xml", "Specify the file extension (json or xml) to be downloaded from NVD") do |file_ext|
-    options[:file_ext] = file_ext || 'json'
-  end
-
-  opts.on_tail("-h", "--help", "Show this message") do
-    puts opts
-    exit
-  end
-end.parse!
-
-files = File.join(CVEServer::Boot.config.raw_data_path, "*.#{options[:file_ext]}.gz")
+files = File.join(CVEServer::Boot.config.raw_data_path, "*seed.json.gz")
 
 CVEServer::Cve.drop_all
 CVEServer::Cpe.drop_all
@@ -31,15 +13,12 @@ CVEServer::CpeWithVersion.drop_all
 Dir.glob(files).sort.each do |infile|
   puts "Uncompressing #{infile}"
   input = Zlib::GzipReader.open(infile).read
-  if options[:file_ext] == 'json'
-    @doc = CVEServer::NVD::JSON::Reader.new(input)
-  else
-    @doc = CVEServer::NVD::Reader.new(input)
-  end
+  @doc = CVEServer::NVD::JSON::Reader.new(input)
   puts 'Exporting data into the CVE collection'
   CVEServer::Cve.bulk_create(@doc.all_cve)
 end
 puts "Reducing the cpe list"
 CVEServer::Cve.reduce_cpes
 puts 'Creating CVE collection Index'
 CVEServer::Cve.create_index(:cve)
+

bin/update

@@ -0,0 +1,18 @@
+#!/usr/bin/env ruby
+$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__)
+require 'zlib'
+require 'cve_server'
+require 'cve_server/nvd/json/reader'
+
+files = File.join(CVEServer::Boot.config.raw_data_path, "*update.json.gz")
+
+Dir.glob(files).sort.each do |infile|
+  puts "Uncompressing #{infile}"
+  input = Zlib::GzipReader.open(infile).read
+  @doc = CVEServer::NVD::JSON::Reader.new(input)
+  puts 'Exporting data into the CVE collection'
+  CVEServer::Cve.bulk_upsert(@doc.all_cve)
+end
+puts "Reducing the cpe list"
+CVEServer::Cve.reduce_cpes
+

lib/cve_server/db/mongo.rb

@@ -35,6 +35,15 @@ def bulk_create(data)
         db[@collection].bulk_write(inserts, ordered: true)
       end
 
+      def bulk_upsert(data)
+        upserts = data.reduce([]) do |ops, chunk|
+          query = { :id => chunk[:id] }
+          update = { '$set' => chunk}
+          ops << {:update_one => {:filter => query, :update => update, :upsert => true }}
+        end
+        db[@collection].bulk_write(upserts, ordered: true)
+      end
+
       def create_index(param)
         db[@collection].indexes.create_one( { param => 1 } )
       end
@@ -44,7 +53,8 @@ def drop_all
       end
 
       def map_reduce(mapper, reducer, options = {})
-        db[@collection].find({}, no_cursor_timeout: true).map_reduce(mapper, reducer, options).execute
+        #db[@collection].find({}, no_cursor_timeout: true).map_reduce(mapper, reducer, options).execute
+        db[@collection].find({}).map_reduce(mapper, reducer, options).execute
       end
 
       def remove_id(record)