From 15b12282250ad1d8a61416efa6a623e288e54dbc Mon Sep 17 00:00:00 2001 From: Troy Schlueter Date: Fri, 26 Mar 2021 16:46:32 +0000 Subject: [PATCH] Updated to use the new API from NVD --- Gemfile | 2 + Gemfile.lock | 5 +- README.md | 28 +- bin/nvd_download_and_seed | 46 ++ bin/nvd_download_and_update | 38 ++ bin/nvd_downloader | 124 ------ bin/seed | 27 +- bin/update | 18 + lib/cve_server/db/mongo.rb | 12 +- lib/cve_server/nvd/entry.rb | 73 ---- lib/cve_server/nvd/json/entry.rb | 20 +- lib/cve_server/nvd/reader.rb | 26 -- lib/nvd_helper.rb | 113 +++++ scripts/install.sh | 15 +- scripts/update_db.sh | 3 +- spec/cve_server/app_spec.rb | 397 ++++++++++-------- spec/cve_server/cve_spec.rb | 64 +-- spec/cve_server/nvd/entry_spec.rb | 26 -- spec/cve_server/nvd/json/entry_spec.rb | 13 +- spec/cve_server/nvd/json/reader_spec.rb | 2 +- spec/cve_server/nvd/reader_spec.rb | 25 -- ...son.gz => partial-nvdcve-1.0_seed.json.gz} | Bin .../nvd_data/partial-nvdcve-2.0-2014.xml.gz | Bin 2577 -> 0 bytes 23 files changed, 537 insertions(+), 540 deletions(-) create mode 100755 bin/nvd_download_and_seed create mode 100755 bin/nvd_download_and_update delete mode 100755 bin/nvd_downloader create mode 100755 bin/update delete mode 100644 lib/cve_server/nvd/entry.rb delete mode 100644 lib/cve_server/nvd/reader.rb create mode 100755 lib/nvd_helper.rb delete mode 100644 spec/cve_server/nvd/entry_spec.rb delete mode 100644 spec/cve_server/nvd/reader_spec.rb rename spec/fixtures/nvd_data/{partial-nvdcve-1.0.json.gz => partial-nvdcve-1.0_seed.json.gz} (100%) delete mode 100644 spec/fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz diff --git a/Gemfile b/Gemfile index 0c968e26..974879dd 100644 --- a/Gemfile +++ b/Gemfile @@ -19,3 +19,5 @@ group :development, :test do gem 'rack-test', '~> 1.1.0' gem 'simplecov', '~> 0.17.0', :require => false end + +gem "tzinfo", "~> 2.0" diff --git a/Gemfile.lock b/Gemfile.lock index 005afbf4..3cfb959c 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -83,6 +83,8 @@ GEM net-scp (>= 1.1.2) net-ssh (>= 2.8.0) tilt (2.0.9) + tzinfo (2.0.4) + concurrent-ruby (~> 1.0) PLATFORMS ruby @@ -101,6 +103,7 @@ DEPENDENCIES simplecov (~> 0.17.0) sinatra (~> 2.0.7) sinatra-contrib (~> 2.0.7) + tzinfo (~> 2.0) BUNDLED WITH - 1.16.1 + 1.17.3 diff --git a/README.md b/README.md index 6f511087..e788187d 100644 --- a/README.md +++ b/README.md @@ -64,30 +64,18 @@ curl --ssl -s https://raw.githubusercontent.com/SpiderLabs/cve_server/master/scr bundle install - 3. Download the raw data from the National Vulnerability Database. The supported - NVD reports are [XML 2.0 (by default) and JSON 1.0 files](https://nvd.nist.gov/vuln/data-feeds). - - ./bin/nvd_downloader - - or - - ./bin/nvd_downloader -f json - - 4. Configure your database. + 3. Configure your database. vi config/database.yml - 5. Create and populate the database for you environment. - - RACK_ENV=development ./bin/seed - - or - - RACK_ENV=development ./bin/seed -f json - - The `-f` flag with the `json` option will populate the database using the experimental JSON reports from NVD and it renames the `score` key to `base_score` in the `cvss` (v2) field, it also includes the `cvssv3` information and some changes for the links in the `references` field. + 4. Download, create and populate the database for your environment from the National Vulnerability Database via the NVD CVE/CPE API. + Note: The new API service is JSON only. + [NVD API URL](https://services.nvd.nist.gov/rest/json/cves/1.0). - 6. Start the server. + RACK_ENV=development ./bin/nvd_download_and_seed + ** The download may take hours to complete ** + + 5. Start the server. RACK_ENV=development puma diff --git a/bin/nvd_download_and_seed b/bin/nvd_download_and_seed new file mode 100755 index 00000000..5e92f160 --- /dev/null +++ b/bin/nvd_download_and_seed @@ -0,0 +1,46 @@ +#!/usr/bin/env ruby +$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) +require 'zlib' +require 'json' +require 'net/http' +require 'tzinfo' +require 'nvd_helper' +require 'cve_server' +require 'cve_server/nvd/json/reader' + +class NvdSeed + include NVDHelper + + def download_data + if local_data_timestamp = last_data_timestamp + print "There are files in the nvd_data directory. Quitting.\n" + exit + else + files = download_pages + update_db(files) + end + end + + + def update_db(file_list) + CVEServer::Cve.drop_all + CVEServer::Cpe.drop_all + CVEServer::CpeWithVersion.drop_all + + file_list.sort.each do |file| + puts "Uncompressing #{file}" + input = Zlib::GzipReader.open(file).read + @doc = CVEServer::NVD::JSON::Reader.new(input) + puts 'Exporting data into the CVE collection' + CVEServer::Cve.bulk_create(@doc.all_cve) + end + puts "Reducing the cpe list" + CVEServer::Cve.reduce_cpes + puts 'Creating CVE collection Index' + CVEServer::Cve.create_index(:cve) + end +end + +nvd_seed = NvdSeed.new +nvd_seed.download_data + diff --git a/bin/nvd_download_and_update b/bin/nvd_download_and_update new file mode 100755 index 00000000..f5575fa1 --- /dev/null +++ b/bin/nvd_download_and_update @@ -0,0 +1,38 @@ +#!/usr/bin/env ruby +$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) +require 'json' +require 'net/http' +require 'tzinfo' +require 'nvd_helper' +require 'cve_server' +require 'cve_server/nvd/json/reader' + +class NvdUpdater + include NVDHelper + + def download_data + if local_data_timestamp = last_data_timestamp + files = download_pages(local_data_timestamp) + update_db(files) + else + print "There are no local files to aquire a timestamp from for updating.\n" + end + end + + + def update_db(file_list) + file_list.sort.each do |file| + puts "Uncompressing #{file}" + input = Zlib::GzipReader.open(file).read + @doc = CVEServer::NVD::JSON::Reader.new(input) + puts 'Exporting data into the CVE collection' + CVEServer::Cve.bulk_upsert(@doc.all_cve) + end + puts "Reducing the cpe list" + CVEServer::Cve.reduce_cpes + end +end + +nvd_updater = NvdUpdater.new +nvd_updater.download_data + diff --git a/bin/nvd_downloader b/bin/nvd_downloader deleted file mode 100755 index b3e4b954..00000000 --- a/bin/nvd_downloader +++ /dev/null @@ -1,124 +0,0 @@ -#!/usr/bin/env ruby -$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) -require 'nokogiri' -require 'net/http' -require 'optparse' -require 'ostruct' -require 'cve_server' - -class NvdDownloader - NVD_FEED_TYPE = %i{json xml} - NVD_FEED_NAMES = %w{modified recent}.push(*2002..Time.new.year).map(&:to_s) - - def initialize(nvd_feed_type:, nvd_feed_names:) - abort "#{nvd_feed_type} format not supported" unless NVD_FEED_TYPE.include? nvd_feed_type - abort "Invalid NVD feed names provided: [#{nvd_feed_names.join(',')}]" if (NVD_FEED_NAMES & nvd_feed_names).empty? - @nvd_feed_type = nvd_feed_type - @nvd_feed_names = nvd_feed_names - # CveServer supports only XML schema version 2.0 and JSON schema version 1.1 NVD Data Feeds - @schema_version = (@nvd_feed_type == :json) ? '1.1' : '2.0' - end - - # @param [Array] args - # @return [Hash{Symbol => Symbol}] - def self.parse(args) - options = OpenStruct.new - options.nvd_feed_type = :json - options.nvd_feed_names = NVD_FEED_NAMES - - opt_parser = OptionParser.new do |opts| - opts.banner = 'Usage: nvd_downloader [options]' - options - - opts.on("-f", "--feed_type TYPE", NVD_FEED_TYPE, - "Specify the NVD source feed type to downloaded (#{NVD_FEED_TYPE.join(',')})") do |nvd_feed_type| - options.nvd_feed_type = nvd_feed_type - end - - opts.on("-n" "--nvd_feed_names [2018,modified]", Array, "Specify a list of NVD feeds to download (#{NVD_FEED_NAMES.join(',')})") do |feed_names| - sanitized = NVD_FEED_NAMES & feed_names.map(&:downcase) - options.nvd_feed_names = sanitized unless sanitized.empty? - end - - opts.on_tail("-h", "--help", "Show this message") do - puts opts - exit - end - end - - opt_parser.parse!(args) - options.to_h - end - - def download(url) - uri = URI.parse(url) - use_ssl = uri.scheme == 'https' - Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http| - http.request_get(uri.path) do |response| - case response - when Net::HTTPSuccess then - return response.body - when Net::HTTPRedirection then - location = response['location'] - fail "redirected to #{location}" - else - fail "Unable to download #{url}" - end - end - end - end - - def dest_path(url) - filename = File.basename(url) - File.join(CVEServer::Boot.config.raw_data_path, filename) - end - - def last_modified_date(meta_content) - match = meta_content.match(/(?:lastModifiedDate:)(?\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}-\d{2}:\d{2})/) - abort "Missing last modified date from meta file" unless match.is_a? MatchData - DateTime.parse(match[:date]) - end - - def gz_to_download - @nvd_feed_names.map do |nvd_feed_name| - base = "https://nvd.nist.gov/feeds/#{@nvd_feed_type}/cve/#{@schema_version}/nvdcve-#{@schema_version}-#{nvd_feed_name}" - meta_url = "#{base}.meta" - local_meta_file = dest_path(meta_url) - gz_url = "#{base}.#{@nvd_feed_type}.gz" - - # If the metadata file doesn't exist, download it and add the gzip to the pending feed download list - downloaded_meta_content = self.download(meta_url) - unless File.exists?(local_meta_file) - File.write(local_meta_file, downloaded_meta_content) - end - - downloaded_mod_date = last_modified_date(downloaded_meta_content) - local_mod_date = last_modified_date(File.read(local_meta_file)) - local_gz = dest_path(gz_url) - if downloaded_mod_date > local_mod_date || !File.exists?(local_gz) - File.delete(local_gz) if File.exists?(local_gz) - File.write(local_meta_file, downloaded_meta_content) - next gz_url - end - - local_extracted = dest_path("#{base}.#{@nvd_feed_type}") - puts "#{File.basename(local_extracted)} is up-to-date, skipping download." - next - end.compact - end - - def download_feeds - gz_urls = gz_to_download - gz_urls.each_with_index do |url, i| - dest = dest_path(url) - puts "Downloading #{i+1} of #{gz_urls.length}: #{dest}" - content = download(url) - File.write(dest, content) - end - end -end - - -options = NvdDownloader.parse(ARGV) -nvd_downloader = NvdDownloader.new(options) -nvd_downloader.download_feeds diff --git a/bin/seed b/bin/seed index dcca779b..9f731069 100755 --- a/bin/seed +++ b/bin/seed @@ -2,27 +2,9 @@ $LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) require 'zlib' require 'cve_server' -require 'cve_server/nvd/reader' require 'cve_server/nvd/json/reader' -require 'optparse' -usage = 'Usage: seed [options]' -options = { file_ext: 'json' } -OptionParser.new do |opts| - opts.banner = usage - - # Mandatory argument. - opts.on("-f", "--file_extension json or xml", "Specify the file extension (json or xml) to be downloaded from NVD") do |file_ext| - options[:file_ext] = file_ext || 'json' - end - - opts.on_tail("-h", "--help", "Show this message") do - puts opts - exit - end -end.parse! - -files = File.join(CVEServer::Boot.config.raw_data_path, "*.#{options[:file_ext]}.gz") +files = File.join(CVEServer::Boot.config.raw_data_path, "*seed.json.gz") CVEServer::Cve.drop_all CVEServer::Cpe.drop_all @@ -31,11 +13,7 @@ CVEServer::CpeWithVersion.drop_all Dir.glob(files).sort.each do |infile| puts "Uncompressing #{infile}" input = Zlib::GzipReader.open(infile).read - if options[:file_ext] == 'json' - @doc = CVEServer::NVD::JSON::Reader.new(input) - else - @doc = CVEServer::NVD::Reader.new(input) - end + @doc = CVEServer::NVD::JSON::Reader.new(input) puts 'Exporting data into the CVE collection' CVEServer::Cve.bulk_create(@doc.all_cve) end @@ -43,3 +21,4 @@ puts "Reducing the cpe list" CVEServer::Cve.reduce_cpes puts 'Creating CVE collection Index' CVEServer::Cve.create_index(:cve) + diff --git a/bin/update b/bin/update new file mode 100755 index 00000000..58d73122 --- /dev/null +++ b/bin/update @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby +$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) +require 'zlib' +require 'cve_server' +require 'cve_server/nvd/json/reader' + +files = File.join(CVEServer::Boot.config.raw_data_path, "*update.json.gz") + +Dir.glob(files).sort.each do |infile| + puts "Uncompressing #{infile}" + input = Zlib::GzipReader.open(infile).read + @doc = CVEServer::NVD::JSON::Reader.new(input) + puts 'Exporting data into the CVE collection' + CVEServer::Cve.bulk_upsert(@doc.all_cve) +end +puts "Reducing the cpe list" +CVEServer::Cve.reduce_cpes + diff --git a/lib/cve_server/db/mongo.rb b/lib/cve_server/db/mongo.rb index 06aeb451..3c0c8d97 100644 --- a/lib/cve_server/db/mongo.rb +++ b/lib/cve_server/db/mongo.rb @@ -35,6 +35,15 @@ def bulk_create(data) db[@collection].bulk_write(inserts, ordered: true) end + def bulk_upsert(data) + upserts = data.reduce([]) do |ops, chunk| + query = { :id => chunk[:id] } + update = { '$set' => chunk} + ops << {:update_one => {:filter => query, :update => update, :upsert => true }} + end + db[@collection].bulk_write(upserts, ordered: true) + end + def create_index(param) db[@collection].indexes.create_one( { param => 1 } ) end @@ -44,7 +53,8 @@ def drop_all end def map_reduce(mapper, reducer, options = {}) - db[@collection].find({}, no_cursor_timeout: true).map_reduce(mapper, reducer, options).execute + #db[@collection].find({}, no_cursor_timeout: true).map_reduce(mapper, reducer, options).execute + db[@collection].find({}).map_reduce(mapper, reducer, options).execute end def remove_id(record) diff --git a/lib/cve_server/nvd/entry.rb b/lib/cve_server/nvd/entry.rb deleted file mode 100644 index 5fca3afb..00000000 --- a/lib/cve_server/nvd/entry.rb +++ /dev/null @@ -1,73 +0,0 @@ -require 'cve_server/nvd/cvss' -module CVEServer - module NVD - # CVEServer::NVD::Entry provides an easy way to parse entries from the - # reports from the National Vulnerability Database. - # - # https://nvd.nist.gov/download.cfm - class Entry - def initialize(entry) - @entry = entry - end - - def to_hash - { - id: @entry['id'], - summary: xpath_content('.//vuln:summary'), - cwe: cwe_id, - published_at: xpath_content('.//vuln:published-datetime'), - updated_at: xpath_content('.//vuln:last-modified-datetime'), - cvss: CVEServer::NVD::Cvss.new(cvss).to_hash, - references: references, - cpes: cpes, - cpes_with_version: cpes_with_version - } - end - - private - - def xpath_content(path, node = @entry) - node.xpath(path)[0].content unless node.xpath(path)[0].nil? - end - - def cwe_id - cwe = @entry.xpath('.//vuln:cwe')[0] - cwe['id'] unless cwe.nil? - end - - def cvss - path = './/cvss:base_metrics/cvss:*' - @entry.search(path).each_with_object({}) do |cvss, h| - h[cvss.name.gsub(/-/, '_').to_sym] = cvss.content - h - end - end - - def references - @entry.xpath('.//vuln:references').collect do |node| - { - type: node['reference_type'], - name: xpath_content('.//vuln:source', node), - href: node.xpath('.//vuln:reference')[0]['href'], - content: xpath_content('.//vuln:reference', node) - } - end - end - - def cpes - path = './/vuln:vulnerable-software-list/vuln:product' - full_cpes = remove_cpe_text(@entry.xpath(path).collect(&:content)) - full_cpes.map {|cpe| cpe.split(":")[0..1].join(":")}.uniq - end - - def cpes_with_version - path = './/vuln:vulnerable-software-list/vuln:product' - remove_cpe_text(@entry.xpath(path).collect(&:content)) - end - - def remove_cpe_text(cpes) - cpes.map {|cpe| cpe.gsub(/^cpe:\/\w:/,"")} - end - end - end -end diff --git a/lib/cve_server/nvd/json/entry.rb b/lib/cve_server/nvd/json/entry.rb index 1ef4b669..88a13a0d 100644 --- a/lib/cve_server/nvd/json/entry.rb +++ b/lib/cve_server/nvd/json/entry.rb @@ -90,15 +90,23 @@ def cpes_affected end def cpes - cpes_with_version.map { |cpe| cpe.split(':')[0..1].join(':') }.uniq + cpes_with_version.map { |cpe| cpe.split(':')[0..1].join(':') unless cpe.nil? }.uniq end def cpes_with_version cpe_regex = /^cpe:(?:\/|2\.[23]:)[aho]:(?[^:]+):(?[^:]+):(?[^:]+)/ + cpe_versionStartIncluding = /versionStartIncluding:(?[^:]+)/ + cpe_versionStartExcluding = /versionStartExcluding:(?[^:]+)/ + cpe_versionEndIncluding = /versionEndIncluding:(?[^:]+)/ + cpe_versionEndExcluding = /versionEndExcluding:(?[^:]+)/ full_cpes.map do |cpe| if match = cpe.match(cpe_regex) cpe_parts = [match[:vendor], match[:product]] cpe_parts << match[:version] unless match[:version] == '*' + cpe_parts << cpe.match(cpe_versionStartIncluding) unless cpe.match(cpe_versionStartIncluding).nil? + cpe_parts << cpe.match(cpe_versionStartExcluding) unless cpe.match(cpe_versionStartExcluding).nil? + cpe_parts << cpe.match(cpe_versionEndIncluding) unless cpe.match(cpe_versionEndIncluding).nil? + cpe_parts << cpe.match(cpe_versionEndExcluding) unless cpe.match(cpe_versionEndExcluding).nil? cpe_parts.join(':') end end.uniq @@ -126,8 +134,14 @@ def nested_cpes(children) children.each do |child| if child.has_key?('cpe_match') child['cpe_match'].each do |cpe_match| - cpes << cpe_match['cpe23Uri'] if cpe_match.has_key?('cpe23Uri') - cpes << cpe_match['cpe22Uri'] if cpe_match.has_key?('cpe22Uri') + cpe_temp = "" + cpe_temp += cpe_match['cpe23Uri'] if cpe_match.has_key?('cpe23Uri') + cpe_temp += cpe_match['cpe22Uri'] if cpe_match.has_key?('cpe22Uri') + cpe_temp += "versionStartIncluding:#{cpe_match['versionStartIncluding']}:" if cpe_match.has_key?('versionStartIncluding') + cpe_temp += "versionStartExcluding:#{cpe_match['versionStartExcluding']}:" if cpe_match.has_key?('versionStartExcluding') + cpe_temp += "versionEndIncluding:#{cpe_match['versionEndIncluding']}:" if cpe_match.has_key?('versionEndIncluding') + cpe_temp += "versionEndExcluding:#{cpe_match['versionEndExcluding']}" if cpe_match.has_key?('versionEndExcluding') + cpes << cpe_temp end elsif child.has_key?('children') cpes.push *nested_cpes(child['children']).flatten diff --git a/lib/cve_server/nvd/reader.rb b/lib/cve_server/nvd/reader.rb deleted file mode 100644 index 28ddfd81..00000000 --- a/lib/cve_server/nvd/reader.rb +++ /dev/null @@ -1,26 +0,0 @@ -require 'nokogiri' -require 'cve_server/nvd/entry' - -module CVEServer - module NVD - # CVEServer::NVD::Reader provides an easy way to read the reports - # (version 2.0) from the National Vulnerability Database. - # - # https://nvd.nist.gov/download.cfm - class Reader - def initialize(input) - @doc = Nokogiri::XML(input) - end - - def all_cve - @doc.xpath('//xmlns:entry').collect do |entry| - CVEServer::NVD::Entry.new(entry).to_hash - end - end - - def each_cve(&blk) - all_cve.each(&blk) - end - end - end -end diff --git a/lib/nvd_helper.rb b/lib/nvd_helper.rb new file mode 100755 index 00000000..95b8d90d --- /dev/null +++ b/lib/nvd_helper.rb @@ -0,0 +1,113 @@ +#!/usr/bin/env ruby + +module NVDHelper + + def valid_json?(json) + JSON.parse(json) + rescue JSON::ParserError => e + return false + end + + def dest_path(page) + filename = "#{Time.now.strftime('%Y-%m-%dT%H-%M-%S_')}#{page}.json.gz" + File.join(CVEServer::Boot.config.raw_data_path, filename) + end + + def last_data_timestamp + latest_file = Dir.glob(CVEServer::Boot.config.raw_data_path+"/*").max_by {|f| File.mtime(f)} + if latest_file.nil? + print "No local files available\n" + return nil + end + print "Newest local file is #{latest_file}\n" + puts "Uncompressing #{latest_file}" + file = Zlib::GzipReader.open(latest_file).read + unless parsed = valid_json?(file) + print "Newest local file fails JSON parsing, continuing as if there are no files." + return nil + end + print "The CVE_data_timestamp is #{parsed["CVE_data_timestamp"]}\n" + return "#{Time.parse(parsed["CVE_data_timestamp"]).strftime("%FT%T:%L")}"+"%20UTC" + end + + def download(url) + uri = URI.parse(url) + use_ssl = uri.scheme == 'https' + Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http| + begin + http.request_get(uri) do |response| + case response + when Net::HTTPSuccess then + return response.body + when Net::HTTPRedirection then + location = response['location'] + fail "redirected to #{location}" + else + fail "Unable to download #{url}" + end + end + rescue + print "Retrying ... #{uri}\n" + sleep(15) + retry + end + end + end + + def download_pages(data_timestamp = nil) + if data_timestamp.nil? + nvd_url_mod = "?" + else + nvd_url_mod = "?modStartDate=#{data_timestamp}&includeMatchStringChange=true&" + end + + nvd_url = "https://services.nvd.nist.gov/rest/json/cves/1.0" + print "Starting download...\n" + nvd_response = download(nvd_url+nvd_url_mod+"resultsPerPage=1") + + unless nvd_parsed = valid_json?(nvd_response) + abort "Unable to parse downloaded page!" + end + + if nvd_parsed["totalResults"] == 0 + print "No new CVE to download!\n" + exit + end + + totalResults = nvd_parsed["totalResults"] + resultPages = totalResults / 1000 + remainingResults = totalResults % 1000 + file_list = [] + + print "Total CVE items to download #{totalResults}.\n" + (0..resultPages).each do |page| + if page*1000+999 >= totalResults + print "Downloading CVE Items #{page*1000} to #{page*1000+remainingResults}\n" + else + print "Downloading CVE Items #{page*1000} to #{page*1000+999}\n" + end + + # NIST firewall rules in place to prevent denial of service attacks + # it is recommended that your application sleeps for several seconds between requests + sleep(15) + + nvd_response = download(nvd_url+nvd_url_mod+"startIndex=#{page*1000}&resultsPerPage=1000") + if nvd_json = valid_json?(nvd_response) + if nvd_url_mod == "?" + local_json_file = dest_path((page*1000).to_s + "_seed") + else + local_json_file = dest_path((page*1000).to_s + "_update") + end + #local_json_file = dest_path(page*1000) + print "Writing file #{local_json_file}...\n" + #File.write(local_json_file, JSON.dump(nvd_json["result"])) + Zlib::GzipWriter.open(local_json_file) do |gz| + gz.write JSON.dump(nvd_json["result"]) + end + file_list << local_json_file + end + end + file_list + end +end + diff --git a/scripts/install.sh b/scripts/install.sh index 9bb09736..55c37f1d 100755 --- a/scripts/install.sh +++ b/scripts/install.sh @@ -65,20 +65,14 @@ clone_git_repository() fi } -download_nvd_reports() +download_and_seed() { - local __program='./bin/nvd_downloader' - ${__program} -} - - -seed_collections() -{ - local __program='./bin/seed' + local __program='./bin/nvd_download_and_seed' export RACK_ENV=$rack_env ${__program} } + start_puma() { local __program='bundle exec puma' @@ -134,8 +128,7 @@ install() { cd $dest_dir install_ruby_bundler install_ruby_gems - download_nvd_reports - seed_collections + download_and_seed start_puma if ps awx| grep puma > /dev/null; then diff --git a/scripts/update_db.sh b/scripts/update_db.sh index 864b9981..42db33fe 100755 --- a/scripts/update_db.sh +++ b/scripts/update_db.sh @@ -4,8 +4,7 @@ APP_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/.." && pwd )" if [ -d "$APP_DIR" ] then killall -9 puma - $APP_DIR/bin/nvd_downloader - $APP_DIR/bin/seed RACK_ENV=production; + $APP_DIR/bin/nvd_download_and_update RACK_ENV=production; export RACK_ENV=production cd $APP_DIR && puma ; else diff --git a/spec/cve_server/app_spec.rb b/spec/cve_server/app_spec.rb index 6b92f523..07536609 100644 --- a/spec/cve_server/app_spec.rb +++ b/spec/cve_server/app_spec.rb @@ -60,7 +60,7 @@ describe CVEServer::App do def app - @app ||= CVEServer::App + @app ||= CVEServer::App.new end describe '/v1/cve/:cve' do @@ -69,7 +69,7 @@ def app "CVE-2014-0001" => { "id" => "CVE-2014-0001", "summary" => "Buffer overflow in client/mysql.cc in Oracle MySQL and MariaDB before 5.5.35 allows remote database servers to cause a denial of service (crash) and possibly execute arbitrary code via a long server version string.", - "cwe" => "CWE-119", + "cwe" => ["CWE-119"], "published_at" => "2014-01-31 23:55:00 UTC", "updated_at" => "2019-04-22 17:48:00 UTC", "cvss" => { @@ -102,12 +102,12 @@ def app ], "cpes_affected" => [ "mariadb:mariadb", - "mysql:mysql", - "oracle:mysql", "redhat:enterprise_linux", "redhat:enterprise_linux_desktop", "redhat:enterprise_linux_server", - "redhat:enterprise_linux_workstation" + "redhat:enterprise_linux_workstation", + "mysql:mysql", + "oracle:mysql" ], "cpes" => [ "mariadb:mariadb", @@ -119,7 +119,7 @@ def app "oracle:mysql" ], "cpes_with_version" => [ - "mariadb:mariadb", + "mariadb:mariadb:versionEndIncluding:5.5.34", "redhat:enterprise_linux:5", "redhat:enterprise_linux:6.0", "redhat:enterprise_linux_desktop:5.0", @@ -185,7 +185,7 @@ def app "CVE-2019-1694" => { "id" => "CVE-2019-1694", "summary" => "A vulnerability in the TCP processing engine of Cisco Adaptive Security Appliance (ASA) Software and Cisco Firepower Threat Defense (FTD) Software could allow an unauthenticated, remote attacker to cause an affected device to reload, resulting in a denial of service (DoS) condition. The vulnerability is due to the improper handling of TCP traffic. An attacker could exploit this vulnerability by sending a specific sequence of packets at a high rate through an affected device. A successful exploit could allow the attacker to temporarily disrupt traffic through the device while it reboots.", - "cwe" => "CWE-399", + "cwe" => ["CWE-399"], "published_at" => "2019-05-03 15:29:00 UTC", "updated_at" => "2019-05-07 16:50:00 UTC", "cvss" => { @@ -217,6 +217,18 @@ def app ], "cpes_affected" => [ "cisco:adaptive_security_appliance_software", + "cisco:asa_5505", + "cisco:asa_5510", + "cisco:asa_5512-x", + "cisco:asa_5515-x", + "cisco:asa_5520", + "cisco:asa_5525-x", + "cisco:asa_5540", + "cisco:asa_5545-x", + "cisco:asa_5550", + "cisco:asa_5555-x", + "cisco:asa_5580", + "cisco:asa_5585-x", "cisco:firepower_threat_defense" ], "cpes" => [ @@ -236,7 +248,11 @@ def app "cisco:firepower_threat_defense" ], "cpes_with_version" => [ - "cisco:adaptive_security_appliance_software", + "cisco:adaptive_security_appliance_software:versionEndExcluding:9.4.4.34", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.5:versionEndExcluding:9.6.4.25", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.7:versionEndExcluding:9.8.4", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.9:versionEndExcluding:9.9.2.50", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.10:versionEndExcluding:9.10.1.17", "cisco:asa_5505:-", "cisco:asa_5510:-", "cisco:asa_5512-x:-", @@ -249,7 +265,8 @@ def app "cisco:asa_5555-x:-", "cisco:asa_5580:-", "cisco:asa_5585-x:-", - "cisco:firepower_threat_defense" + "cisco:firepower_threat_defense:versionEndExcluding:6.2.3.12", + "cisco:firepower_threat_defense:versionStartIncluding:6.3.0:versionEndExcluding:6.3.0.3" ] } } @@ -271,7 +288,7 @@ def app "CVE-2014-0001" => [{ "id" => "CVE-2014-0001", "summary" => "Buffer overflow in client/mysql.cc in Oracle MySQL and MariaDB before 5.5.35 allows remote database servers to cause a denial of service (crash) and possibly execute arbitrary code via a long server version string.", - "cwe" => "CWE-119", + "cwe" => ["CWE-119"], "published_at" => "2014-01-31 23:55:00 UTC", "updated_at" => "2019-04-22 17:48:00 UTC", "cvss" => { @@ -304,12 +321,12 @@ def app ], "cpes_affected" => [ "mariadb:mariadb", - "mysql:mysql", - "oracle:mysql", "redhat:enterprise_linux", "redhat:enterprise_linux_desktop", "redhat:enterprise_linux_server", - "redhat:enterprise_linux_workstation" + "redhat:enterprise_linux_workstation", + "mysql:mysql", + "oracle:mysql" ], "cpes" => [ "mariadb:mariadb", @@ -321,7 +338,7 @@ def app "oracle:mysql" ], "cpes_with_version" => [ - "mariadb:mariadb", + "mariadb:mariadb:versionEndIncluding:5.5.34", "redhat:enterprise_linux:5", "redhat:enterprise_linux:6.0", "redhat:enterprise_linux_desktop:5.0", @@ -387,7 +404,7 @@ def app "CVE-2019-1694" => [{ "id" => "CVE-2019-1694", "summary" => "A vulnerability in the TCP processing engine of Cisco Adaptive Security Appliance (ASA) Software and Cisco Firepower Threat Defense (FTD) Software could allow an unauthenticated, remote attacker to cause an affected device to reload, resulting in a denial of service (DoS) condition. The vulnerability is due to the improper handling of TCP traffic. An attacker could exploit this vulnerability by sending a specific sequence of packets at a high rate through an affected device. A successful exploit could allow the attacker to temporarily disrupt traffic through the device while it reboots.", - "cwe" => "CWE-399", + "cwe" => ["CWE-399"], "published_at" => "2019-05-03 15:29:00 UTC", "updated_at" => "2019-05-07 16:50:00 UTC", "cvss" => { @@ -419,6 +436,18 @@ def app ], "cpes_affected" => [ "cisco:adaptive_security_appliance_software", + "cisco:asa_5505", + "cisco:asa_5510", + "cisco:asa_5512-x", + "cisco:asa_5515-x", + "cisco:asa_5520", + "cisco:asa_5525-x", + "cisco:asa_5540", + "cisco:asa_5545-x", + "cisco:asa_5550", + "cisco:asa_5555-x", + "cisco:asa_5580", + "cisco:asa_5585-x", "cisco:firepower_threat_defense" ], "cpes" => [ @@ -438,7 +467,11 @@ def app "cisco:firepower_threat_defense" ], "cpes_with_version" => [ - "cisco:adaptive_security_appliance_software", + "cisco:adaptive_security_appliance_software:versionEndExcluding:9.4.4.34", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.5:versionEndExcluding:9.6.4.25", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.7:versionEndExcluding:9.8.4", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.9:versionEndExcluding:9.9.2.50", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.10:versionEndExcluding:9.10.1.17", "cisco:asa_5505:-", "cisco:asa_5510:-", "cisco:asa_5512-x:-", @@ -451,7 +484,8 @@ def app "cisco:asa_5555-x:-", "cisco:asa_5580:-", "cisco:asa_5585-x:-", - "cisco:firepower_threat_defense" + "cisco:firepower_threat_defense:versionEndExcluding:6.2.3.12", + "cisco:firepower_threat_defense:versionStartIncluding:6.3.0:versionEndExcluding:6.3.0.3" ] }] } @@ -583,32 +617,44 @@ def app describe "/v1/cpes_affected" do url = "/v1/cpes_affected" expected_results = [ - "apache:camel", - "cisco:adaptive_security_appliance_software", - "cisco:firepower_threat_defense", - "mariadb:mariadb", - "microsoft:edge", - "microsoft:excel", - "microsoft:excel_viewer", - "microsoft:internet_explorer", - "microsoft:office", - "microsoft:office_compatibility_pack", - "microsoft:sharepoint_enterprise_server", - "microsoft:sharepoint_server", - "microsoft:windows_10", - "microsoft:windows_7", - "microsoft:windows_8.1", - "microsoft:windows_rt_8.1", - "microsoft:windows_server_2008", - "microsoft:windows_server_2012", - "microsoft:windows_server_2016", - "microsoft:windows_vista", - "microsoft:word", - "mysql:mysql", - "oracle:mysql", + "apache:camel", + "cisco:adaptive_security_appliance_software", + "cisco:asa_5505", + "cisco:asa_5510", + "cisco:asa_5512-x", + "cisco:asa_5515-x", + "cisco:asa_5520", + "cisco:asa_5525-x", + "cisco:asa_5540", + "cisco:asa_5545-x", + "cisco:asa_5550", + "cisco:asa_5555-x", + "cisco:asa_5580", + "cisco:asa_5585-x", + "cisco:firepower_threat_defense", + "mariadb:mariadb", + "microsoft:edge", + "microsoft:excel", + "microsoft:excel_viewer", + "microsoft:internet_explorer", + "microsoft:office", + "microsoft:office_compatibility_pack", + "microsoft:sharepoint_enterprise_server", + "microsoft:sharepoint_server", + "microsoft:windows_10", + "microsoft:windows_7", + "microsoft:windows_8.1", + "microsoft:windows_rt_8.1", + "microsoft:windows_server_2008", + "microsoft:windows_server_2012", + "microsoft:windows_server_2016", + "microsoft:windows_vista", + "microsoft:word", + "mysql:mysql", + "oracle:mysql", "redhat:enterprise_linux", - "redhat:enterprise_linux_desktop", - "redhat:enterprise_linux_server", + "redhat:enterprise_linux_desktop", + "redhat:enterprise_linux_server", "redhat:enterprise_linux_workstation" ] @@ -618,137 +664,142 @@ def app describe "/v1/cpe_with_version" do url = "/v1/cpe_with_version" expected_results = [ - "apache:camel", - "apache:camel:1.0.0", - "apache:camel:1.1.0", - "apache:camel:1.2.0", - "apache:camel:1.3.0", - "apache:camel:1.4.0", - "apache:camel:1.5.0", - "apache:camel:1.6.0", - "apache:camel:1.6.1", - "apache:camel:1.6.2", - "apache:camel:1.6.3", - "apache:camel:1.6.4", - "apache:camel:2.0.0", - "apache:camel:2.1.0", - "apache:camel:2.10.0", - "apache:camel:2.10.1", - "apache:camel:2.10.2", - "apache:camel:2.10.3", - "apache:camel:2.10.4", - "apache:camel:2.10.5", - "apache:camel:2.10.6", - "apache:camel:2.10.7", - "apache:camel:2.11.0", - "apache:camel:2.11.1", - "apache:camel:2.11.2", - "apache:camel:2.12.0", - "apache:camel:2.12.1", - "apache:camel:2.12.2", - "cisco:adaptive_security_appliance_software", - "cisco:asa_5505:-", - "cisco:asa_5510:-", - "cisco:asa_5512-x:-", - "cisco:asa_5515-x:-", - "cisco:asa_5520:-", - "cisco:asa_5525-x:-", - "cisco:asa_5540:-", - "cisco:asa_5545-x:-", - "cisco:asa_5550:-", - "cisco:asa_5555-x:-", - "cisco:asa_5580:-", - "cisco:asa_5585-x:-", - "cisco:firepower_threat_defense", - "mariadb:mariadb", - "microsoft:edge", - "microsoft:edge:-", - "microsoft:excel:2007", - "microsoft:excel_viewer", - "microsoft:internet_explorer:10", - "microsoft:internet_explorer:11", - "microsoft:internet_explorer:9", - "microsoft:office:2010", - "microsoft:office_compatibility_pack", - "microsoft:sharepoint_enterprise_server:2016", - "microsoft:sharepoint_server:2007", - "microsoft:windows_10", - "microsoft:windows_10:-", - "microsoft:windows_10:1511", - "microsoft:windows_10:1607", - "microsoft:windows_7", - "microsoft:windows_7:-", - "microsoft:windows_8.1", - "microsoft:windows_rt_8.1", - "microsoft:windows_server_2008", - "microsoft:windows_server_2008:-", - "microsoft:windows_server_2008:r2", - "microsoft:windows_server_2012:-", - "microsoft:windows_server_2012:r2", - "microsoft:windows_server_2016", - "microsoft:windows_server_2016:-", - "microsoft:windows_vista", - "microsoft:windows_vista:-", - "microsoft:word:2016", - "mysql:mysql:5.5.0", - "mysql:mysql:5.5.1", - "mysql:mysql:5.5.2", - "mysql:mysql:5.5.3", - "mysql:mysql:5.5.4", - "mysql:mysql:5.5.5", - "mysql:mysql:5.5.6", - "mysql:mysql:5.5.7", - "mysql:mysql:5.5.8", - "mysql:mysql:5.5.9", - "oracle:mysql:5.5.10", - "oracle:mysql:5.5.11", - "oracle:mysql:5.5.12", - "oracle:mysql:5.5.13", - "oracle:mysql:5.5.14", - "oracle:mysql:5.5.15", - "oracle:mysql:5.5.16", - "oracle:mysql:5.5.17", - "oracle:mysql:5.5.18", - "oracle:mysql:5.5.19", - "oracle:mysql:5.5.20", - "oracle:mysql:5.5.21", - "oracle:mysql:5.5.22", - "oracle:mysql:5.5.23", - "oracle:mysql:5.5.24", - "oracle:mysql:5.5.25", - "oracle:mysql:5.5.26", - "oracle:mysql:5.5.27", - "oracle:mysql:5.5.28", - "oracle:mysql:5.5.29", - "oracle:mysql:5.5.30", - "oracle:mysql:5.5.31", - "oracle:mysql:5.5.32", - "oracle:mysql:5.5.33", - "oracle:mysql:5.5.34", - "oracle:mysql:5.5.35", - "oracle:mysql:5.5.36", - "oracle:mysql:5.6.0", - "oracle:mysql:5.6.1", - "oracle:mysql:5.6.10", - "oracle:mysql:5.6.11", - "oracle:mysql:5.6.12", - "oracle:mysql:5.6.13", - "oracle:mysql:5.6.14", - "oracle:mysql:5.6.15", - "oracle:mysql:5.6.16", - "oracle:mysql:5.6.2", - "oracle:mysql:5.6.3", - "oracle:mysql:5.6.4", - "oracle:mysql:5.6.5", - "oracle:mysql:5.6.6", - "oracle:mysql:5.6.7", - "oracle:mysql:5.6.8", - "oracle:mysql:5.6.9", - "redhat:enterprise_linux:5", - "redhat:enterprise_linux:6.0", - "redhat:enterprise_linux_desktop:5.0", - "redhat:enterprise_linux_desktop:6.0", - "redhat:enterprise_linux_server:6.0", + "apache:camel:1.0.0", + "apache:camel:1.1.0", + "apache:camel:1.2.0", + "apache:camel:1.3.0", + "apache:camel:1.4.0", + "apache:camel:1.5.0", + "apache:camel:1.6.0", + "apache:camel:1.6.1", + "apache:camel:1.6.2", + "apache:camel:1.6.3", + "apache:camel:1.6.4", + "apache:camel:2.0.0", + "apache:camel:2.1.0", + "apache:camel:2.10.0", + "apache:camel:2.10.1", + "apache:camel:2.10.2", + "apache:camel:2.10.3", + "apache:camel:2.10.4", + "apache:camel:2.10.5", + "apache:camel:2.10.6", + "apache:camel:2.10.7", + "apache:camel:2.11.0", + "apache:camel:2.11.1", + "apache:camel:2.11.2", + "apache:camel:2.12.0", + "apache:camel:2.12.1", + "apache:camel:2.12.2", + "apache:camel:versionEndIncluding:2.11.3", + "cisco:adaptive_security_appliance_software:versionEndExcluding:9.4.4.34", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.10:versionEndExcluding:9.10.1.17", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.5:versionEndExcluding:9.6.4.25", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.7:versionEndExcluding:9.8.4", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.9:versionEndExcluding:9.9.2.50", + "cisco:asa_5505:-", + "cisco:asa_5510:-", + "cisco:asa_5512-x:-", + "cisco:asa_5515-x:-", + "cisco:asa_5520:-", + "cisco:asa_5525-x:-", + "cisco:asa_5540:-", + "cisco:asa_5545-x:-", + "cisco:asa_5550:-", + "cisco:asa_5555-x:-", + "cisco:asa_5580:-", + "cisco:asa_5585-x:-", + "cisco:firepower_threat_defense:versionEndExcluding:6.2.3.12", + "cisco:firepower_threat_defense:versionStartIncluding:6.3.0:versionEndExcluding:6.3.0.3", + "mariadb:mariadb:versionEndIncluding:5.5.34", + "microsoft:edge", + "microsoft:edge:-", + "microsoft:excel:2007", + "microsoft:excel_viewer", + "microsoft:internet_explorer:10", + "microsoft:internet_explorer:11", + "microsoft:internet_explorer:9", + "microsoft:office:2010", + "microsoft:office_compatibility_pack", + "microsoft:sharepoint_enterprise_server:2016", + "microsoft:sharepoint_server:2007", + "microsoft:windows_10", + "microsoft:windows_10:-", + "microsoft:windows_10:1511", + "microsoft:windows_10:1607", + "microsoft:windows_7", + "microsoft:windows_7:-", + "microsoft:windows_8.1", + "microsoft:windows_rt_8.1", + "microsoft:windows_server_2008", + "microsoft:windows_server_2008:-", + "microsoft:windows_server_2008:r2", + "microsoft:windows_server_2012:-", + "microsoft:windows_server_2012:r2", + "microsoft:windows_server_2016", + "microsoft:windows_server_2016:-", + "microsoft:windows_vista", + "microsoft:windows_vista:-", + "microsoft:word:2016", + "mysql:mysql:5.5.0", + "mysql:mysql:5.5.1", + "mysql:mysql:5.5.2", + "mysql:mysql:5.5.3", + "mysql:mysql:5.5.4", + "mysql:mysql:5.5.5", + "mysql:mysql:5.5.6", + "mysql:mysql:5.5.7", + "mysql:mysql:5.5.8", + "mysql:mysql:5.5.9", + "oracle:mysql:5.5.10", + "oracle:mysql:5.5.11", + "oracle:mysql:5.5.12", + "oracle:mysql:5.5.13", + "oracle:mysql:5.5.14", + "oracle:mysql:5.5.15", + "oracle:mysql:5.5.16", + "oracle:mysql:5.5.17", + "oracle:mysql:5.5.18", + "oracle:mysql:5.5.19", + "oracle:mysql:5.5.20", + "oracle:mysql:5.5.21", + "oracle:mysql:5.5.22", + "oracle:mysql:5.5.23", + "oracle:mysql:5.5.24", + "oracle:mysql:5.5.25", + "oracle:mysql:5.5.26", + "oracle:mysql:5.5.27", + "oracle:mysql:5.5.28", + "oracle:mysql:5.5.29", + "oracle:mysql:5.5.30", + "oracle:mysql:5.5.31", + "oracle:mysql:5.5.32", + "oracle:mysql:5.5.33", + "oracle:mysql:5.5.34", + "oracle:mysql:5.5.35", + "oracle:mysql:5.5.36", + "oracle:mysql:5.6.0", + "oracle:mysql:5.6.1", + "oracle:mysql:5.6.10", + "oracle:mysql:5.6.11", + "oracle:mysql:5.6.12", + "oracle:mysql:5.6.13", + "oracle:mysql:5.6.14", + "oracle:mysql:5.6.15", + "oracle:mysql:5.6.16", + "oracle:mysql:5.6.2", + "oracle:mysql:5.6.3", + "oracle:mysql:5.6.4", + "oracle:mysql:5.6.5", + "oracle:mysql:5.6.6", + "oracle:mysql:5.6.7", + "oracle:mysql:5.6.8", + "oracle:mysql:5.6.9", + "redhat:enterprise_linux:5", + "redhat:enterprise_linux:6.0", + "redhat:enterprise_linux_desktop:5.0", + "redhat:enterprise_linux_desktop:6.0", + "redhat:enterprise_linux_server:6.0", "redhat:enterprise_linux_workstation:6.0" ] diff --git a/spec/cve_server/cve_spec.rb b/spec/cve_server/cve_spec.rb index cc90fe15..8b5e37c3 100644 --- a/spec/cve_server/cve_spec.rb +++ b/spec/cve_server/cve_spec.rb @@ -6,32 +6,44 @@ context 'when no CPEs are provided' do it 'should return the expected affected CPEs' do expected_results = [ - "apache:camel", - "cisco:adaptive_security_appliance_software", - "cisco:firepower_threat_defense", - "mariadb:mariadb", - "microsoft:edge", - "microsoft:excel", - "microsoft:excel_viewer", - "microsoft:internet_explorer", - "microsoft:office", - "microsoft:office_compatibility_pack", - "microsoft:sharepoint_enterprise_server", - "microsoft:sharepoint_server", - "microsoft:windows_10", - "microsoft:windows_7", - "microsoft:windows_8.1", - "microsoft:windows_rt_8.1", - "microsoft:windows_server_2008", - "microsoft:windows_server_2012", - "microsoft:windows_server_2016", - "microsoft:windows_vista", - "microsoft:word", - "mysql:mysql", - "oracle:mysql", - "redhat:enterprise_linux", - "redhat:enterprise_linux_desktop", - "redhat:enterprise_linux_server", + "apache:camel", + "cisco:adaptive_security_appliance_software", + "cisco:asa_5505", + "cisco:asa_5510", + "cisco:asa_5512-x", + "cisco:asa_5515-x", + "cisco:asa_5520", + "cisco:asa_5525-x", + "cisco:asa_5540", + "cisco:asa_5545-x", + "cisco:asa_5550", + "cisco:asa_5555-x", + "cisco:asa_5580", + "cisco:asa_5585-x", + "cisco:firepower_threat_defense", + "mariadb:mariadb", + "microsoft:edge", + "microsoft:excel", + "microsoft:excel_viewer", + "microsoft:internet_explorer", + "microsoft:office", + "microsoft:office_compatibility_pack", + "microsoft:sharepoint_enterprise_server", + "microsoft:sharepoint_server", + "microsoft:windows_10", + "microsoft:windows_7", + "microsoft:windows_8.1", + "microsoft:windows_rt_8.1", + "microsoft:windows_server_2008", + "microsoft:windows_server_2012", + "microsoft:windows_server_2016", + "microsoft:windows_vista", + "microsoft:word", + "mysql:mysql", + "oracle:mysql", + "redhat:enterprise_linux", + "redhat:enterprise_linux_desktop", + "redhat:enterprise_linux_server", "redhat:enterprise_linux_workstation" ] expect(described_class.all_cpes_affected).to match_array(expected_results) diff --git a/spec/cve_server/nvd/entry_spec.rb b/spec/cve_server/nvd/entry_spec.rb deleted file mode 100644 index 6c5637c7..00000000 --- a/spec/cve_server/nvd/entry_spec.rb +++ /dev/null @@ -1,26 +0,0 @@ -require 'spec_helper' -require 'cve_server/nvd/entry' -require 'zlib' -require 'nokogiri' - -describe CVEServer::NVD::Entry do - before :all do - @infile = File.expand_path('../../../fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz', __FILE__) - @xml = Zlib::GzipReader.open(@infile).read - @doc = Nokogiri::XML(@xml) - @entry = @doc.xpath('//xmlns:entry').first - @nvd_entry = CVEServer::NVD::Entry.new(@entry) - end - - it 'should be an instance of CVEServer::NVD::Entry' do - expect(@nvd_entry).to be_an_instance_of(CVEServer::NVD::Entry) - end - - it 'should have the CVE attributes' do - expect(@nvd_entry.to_hash.keys).to include(:id, :summary, :cwe, :published_at, :updated_at, :cvss, :references, :cpes) - end - - it 'should have the CVE ID CVE-2014-0001' do - expect(@nvd_entry.to_hash[:id]).to eq 'CVE-2014-0001' - end -end diff --git a/spec/cve_server/nvd/json/entry_spec.rb b/spec/cve_server/nvd/json/entry_spec.rb index 74d51944..fab68f3f 100644 --- a/spec/cve_server/nvd/json/entry_spec.rb +++ b/spec/cve_server/nvd/json/entry_spec.rb @@ -5,7 +5,7 @@ describe CVEServer::NVD::JSON::Entry do context 'when it receives a valid entry from the NVD dataset' do let(:infile) { - json_file = '../../../../fixtures/nvd_data/partial-nvdcve-1.0.json.gz' + json_file = '../../../../fixtures/nvd_data/partial-nvdcve-1.0_seed.json.gz' File.expand_path(json_file, __FILE__) } let(:input) { Zlib::GzipReader.open(infile).read } @@ -36,7 +36,7 @@ describe '#cwe' do it 'should return the CWE-476' do expect(subject.cwe).not_to be_nil - expect(subject.cwe).to eq('CWE-476') + expect(subject.cwe).to eq(['CWE-476']) end end @@ -251,7 +251,11 @@ describe '#cpes_with_version' do it 'should return all CPEs with versions' do expected_cpes = [ - "cisco:adaptive_security_appliance_software", + "cisco:adaptive_security_appliance_software:versionEndExcluding:9.4.4.34", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.5:versionEndExcluding:9.6.4.25", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.7:versionEndExcluding:9.8.4", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.9:versionEndExcluding:9.9.2.50", + "cisco:adaptive_security_appliance_software:versionStartIncluding:9.10:versionEndExcluding:9.10.1.17", "cisco:asa_5505:-", "cisco:asa_5510:-", "cisco:asa_5512-x:-", @@ -264,7 +268,8 @@ "cisco:asa_5555-x:-", "cisco:asa_5580:-", "cisco:asa_5585-x:-", - "cisco:firepower_threat_defense" + "cisco:firepower_threat_defense:versionEndExcluding:6.2.3.12", + "cisco:firepower_threat_defense:versionStartIncluding:6.3.0:versionEndExcluding:6.3.0.3" ] expect(subject.cpes_with_version).not_to be_nil expect(subject.cpes_with_version).to eq(expected_cpes) diff --git a/spec/cve_server/nvd/json/reader_spec.rb b/spec/cve_server/nvd/json/reader_spec.rb index 651be407..72dc2143 100644 --- a/spec/cve_server/nvd/json/reader_spec.rb +++ b/spec/cve_server/nvd/json/reader_spec.rb @@ -5,7 +5,7 @@ describe CVEServer::NVD::JSON::Reader do context 'when it receives a valid JSON input from NVD' do let(:infile) { - json_file = '../../../../fixtures/nvd_data/partial-nvdcve-1.0.json.gz' + json_file = '../../../../fixtures/nvd_data/partial-nvdcve-1.0_seed.json.gz' File.expand_path(json_file, __FILE__) } let(:input) { Zlib::GzipReader.open(infile).read } diff --git a/spec/cve_server/nvd/reader_spec.rb b/spec/cve_server/nvd/reader_spec.rb deleted file mode 100644 index 2e87afc8..00000000 --- a/spec/cve_server/nvd/reader_spec.rb +++ /dev/null @@ -1,25 +0,0 @@ -require 'spec_helper' -require 'cve_server/nvd/reader' -require 'zlib' - -describe CVEServer::NVD::Reader do - before :all do - @infile = File.expand_path('../../../fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz', __FILE__) - @xml = Zlib::GzipReader.open(@infile).read - @nvd_reader = CVEServer::NVD::Reader.new(@xml) - end - - it 'Should return an array with CVEs' do - expect(@nvd_reader.all_cve).to be_a(Array) - end - - it 'Should return 5 entries for CVEs' do - expect(@nvd_reader.all_cve.size).to eq 5 - end - - it 'Should pass each CVE as a Hash' do - @nvd_reader.each_cve do |cve| - expect(cve).to be_an_instance_of(Hash) - end - end -end diff --git a/spec/fixtures/nvd_data/partial-nvdcve-1.0.json.gz b/spec/fixtures/nvd_data/partial-nvdcve-1.0_seed.json.gz similarity index 100% rename from spec/fixtures/nvd_data/partial-nvdcve-1.0.json.gz rename to spec/fixtures/nvd_data/partial-nvdcve-1.0_seed.json.gz diff --git a/spec/fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz b/spec/fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz deleted file mode 100644 index afbec3e382066d20aea3f7aeb0bdcd975caf9bdf..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2577 zcmV+s3hwnEiwFo)MLAXg18`w-bZKF1EpB#XV|HaNGA=MJGB7bTE_iKh0PS0CbK1HR z{@!1q$;_FyGqr5lm={k%CogoG+a&Y`n&#dw8DR_1iLp7BfaH9*zx}RcoA4G=APH$s zW;!KUyKC)gwfj7>+5LRe3(=4zg2&O`dxEX^h(&%J@Tjx*{zvnRQGWlq`ffKG1_&f0 z;R&Di4L?rU-cDD_zGs^13XZstxDyZITbK2yITkZ5Om;F|e<;LqH6>vTZBVjvrQslq zmMbXwk@+f7R+b9Q+DZb*u_ra&cd#_aMD>o%%u^zCu$;Y%Mne$ zD$fn&QvD(qUCS{n_{+-YHm^2eG^9crJ*dcSK5w{aI98K5o(r}e>=sIf zRk$;JkLr|rLfoY(eDPXTu&W-E3HQZhI;ky^C7pZXSL?f z`uVp^VR~$_q96DAA-jPxQ9Y^uoK~C|n^hjjF6?5S*0<_e{j8R&PLIs!>BJ4xI8~L& z;JrTVm+JTZ^XBpXNoL|6p&3&=lB|>5Yh-SC)`KDCVOAg)TUaD9yF&3G@mY021hlZ_&RgO>jsDk>Bl82rHB3|8%7>pruF(Ai~sNO(;8N!iHQmVAFOj8@ojp7gJDj zH|^@(_KZka5~uNbcDxQ5?_2{C9q z=Ax*rWZ*XpPHWKln};FA2@ASZY9B>69$moz_B;GD>=9zQ1-D|~7oHVNY%M_b#>LUW z>RVA^tXK?T?G-#GmR%xqdn>)VHP24>&yLQIFZS1Su1BLF;X|4(8uJILX_z#CjUOkR ziO(9yogQ5@_BAH)EW0urfa%e#1>(GR^mV_vdKi*!G?}?9ffhr}^RJmt0I13p+?Bm> z*6*>1r$k<(-^GXhF1|v)OU{FSmz-DVcd_uG-&+I#YnJet^6e}v-|MWV81w)J+*S_; zZ5T%JkR|Of9w8nfKjZ-5O-(AX@2gAog!&;vr?-uNpCBl@=u|<7qXX1pZ6G#C6G#`) z5KIUNOfQxU0mh{Y&?3PSb@5e&$R(Olp z4HR`SMUxhn3CtJyalp`!Q?L=nQ75yVpUWWu&^hYhJkXhw>fplECl^c3Fznyq41)rd z=`zoUYoTXjLa?*SS;maIg)Of~);2(tfhRo(dC8%-vAsd)iT%W(`}H%V1v$0CS<=Fk zT326Z)cQA9YqF8{!h3Ia29{4~g)=b6uu0RhJg4lr6$}-A>ZodiGq8d|D#~!d^<09> z#CbaddpiSr+%qtpyTnq*td@<-ww0&Y)1HCZ#lo6rU`tWk!ST_0%rqyX?bsg(jaXYe zFpF-*BC`-qUwnOd+$q@);c*aojee7d{eIH*zNnoY)z7c?k1mcI_4EHc^7*drmN;#l znwaFNZrq)_U5!oqdQXA|k?1pDcVU#) z?Ue9D=<@Ug1yE6XYL?Ju)R8yP9W}&mBugToRI2aDTOe3|NORmADd9f{k*~jwxa?|` zRB{}iPw;1`inE$=cDGt?ww4NixV)@=L|ljgTZ?2lzah!mm?ahQ`r6OH6kc%XH{!QX zPf!1O97`wjOQTrM#uUqaB8pY9-ze4_#d%)UOvNI2>pU``V~HcmPstDZHRyW3%J)ffKnEqFFau*fT`+X zPRP`=oAG0ryBR3}^1=StbTr(}QSK6OyW~x}fa%YKg)t32YMh&+T&PH+IJri!Dcf!C zBMGbvzVJ>*ndK35=L9+}y4Gf7$rWEq115o^$qJu#P9Uq~?#^IoAyTy{d7G(Yn z9t17GhXg2zf{5#b=F~`K+QvA0JAudf1<15}PO}ZB?*Ue4O7D&Om8` zi;nXwP?|gwls>H;9sf9eL(|(u(`9wkPYU`mRk`%i(e#r61hD`i{59bIwZcK_M_i_A zNMX@c?-;BA02cM>O>w}>| nfZZTr;u9Kmx!-+q&N9bB(xShNh3u+sTz&U{8W=;`!$$xBANB{B