diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..9999d1f6 --- /dev/null +++ b/.gitignore @@ -0,0 +1,9 @@ +db/* +!db/.gitkeep +nvd_data/* +!nvd_data/.gitkeep +log/* +!log/.gitkeep +tmp/* +!tmp/.gitkeep +coverage diff --git a/.rspec b/.rspec new file mode 100644 index 00000000..c5b8214a --- /dev/null +++ b/.rspec @@ -0,0 +1,3 @@ +--colour +--backtrace +--format documentation diff --git a/.ruby-gemset b/.ruby-gemset new file mode 100644 index 00000000..d3ea42b6 --- /dev/null +++ b/.ruby-gemset @@ -0,0 +1 @@ +cve_server diff --git a/.ruby-version b/.ruby-version new file mode 100644 index 00000000..8274681f --- /dev/null +++ b/.ruby-version @@ -0,0 +1 @@ +ruby-2.2.3 diff --git a/Capfile b/Capfile new file mode 100644 index 00000000..12995210 --- /dev/null +++ b/Capfile @@ -0,0 +1,8 @@ +require 'capistrano/setup' +require 'capistrano/deploy' +require 'capistrano/puma' +require 'capistrano/puma' +require 'capistrano/rvm' +require 'capistrano/bundler' + +Dir.glob('lib/capistrano/tasks/*.rake').each { |r| import r } diff --git a/Gemfile b/Gemfile new file mode 100644 index 00000000..21a56a4a --- /dev/null +++ b/Gemfile @@ -0,0 +1,21 @@ +source 'https://rubygems.org' + +gem 'sinatra', '~> 1.4.6' +gem 'sinatra-json', '~> 0.1.0' +gem 'nokogiri', '~> 1.6.6.2' +gem 'mongo', '~> 2.1.0' +gem 'puma', '~> 2.13.4' + +group :production do + gem 'capistrano', '~> 3.4.0' + gem 'capistrano-bundler', '~> 1.1.4' + gem 'capistrano-rvm', '~> 0.1.2' + gem 'capistrano3-puma', '~> 1.2.1' +end + +group :development, :test do + gem 'pry', '~> 0.10.1' + gem 'rspec', '~> 3.3.0' + gem 'rack-test', '~> 0.6.3' + gem 'simplecov', '~> 0.10.0', :require => false +end diff --git a/Gemfile.lock b/Gemfile.lock new file mode 100644 index 00000000..8b37b435 --- /dev/null +++ b/Gemfile.lock @@ -0,0 +1,96 @@ +GEM + remote: https://rubygems.org/ + specs: + bson (3.2.4) + capistrano (3.4.0) + i18n + rake (>= 10.0.0) + sshkit (~> 1.3) + capistrano-bundler (1.1.4) + capistrano (~> 3.1) + sshkit (~> 1.2) + capistrano-rvm (0.1.2) + capistrano (~> 3.0) + sshkit (~> 1.2) + capistrano3-puma (1.2.1) + capistrano (~> 3.0) + puma (>= 2.6) + coderay (1.1.0) + colorize (0.7.7) + diff-lcs (1.2.5) + docile (1.1.5) + i18n (0.7.0) + json (1.8.3) + method_source (0.8.2) + mini_portile (0.6.2) + mongo (2.1.0) + bson (~> 3.0) + multi_json (1.11.1) + net-scp (1.2.1) + net-ssh (>= 2.6.5) + net-ssh (2.9.2) + nokogiri (1.6.6.2) + mini_portile (~> 0.6.0) + pry (0.10.1) + coderay (~> 1.1.0) + method_source (~> 0.8.1) + slop (~> 3.4) + puma (2.13.4) + rack (1.6.2) + rack-protection (1.5.3) + rack + rack-test (0.6.3) + rack (>= 1.0) + rake (10.4.2) + rspec (3.3.0) + rspec-core (~> 3.3.0) + rspec-expectations (~> 3.3.0) + rspec-mocks (~> 3.3.0) + rspec-core (3.3.1) + rspec-support (~> 3.3.0) + rspec-expectations (3.3.0) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.3.0) + rspec-mocks (3.3.1) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.3.0) + rspec-support (3.3.0) + simplecov (0.10.0) + docile (~> 1.1.0) + json (~> 1.8) + simplecov-html (~> 0.10.0) + simplecov-html (0.10.0) + sinatra (1.4.6) + rack (~> 1.4) + rack-protection (~> 1.4) + tilt (>= 1.3, < 3) + sinatra-json (0.1.0) + multi_json (~> 1.0) + sinatra (~> 1.0) + slop (3.6.0) + sshkit (1.7.1) + colorize (>= 0.7.0) + net-scp (>= 1.1.2) + net-ssh (>= 2.8.0) + tilt (2.0.1) + +PLATFORMS + ruby + +DEPENDENCIES + capistrano (~> 3.4.0) + capistrano-bundler (~> 1.1.4) + capistrano-rvm (~> 0.1.2) + capistrano3-puma (~> 1.2.1) + mongo (~> 2.1.0) + nokogiri (~> 1.6.6.2) + pry (~> 0.10.1) + puma (~> 2.13.4) + rack-test (~> 0.6.3) + rspec (~> 3.3.0) + simplecov (~> 0.10.0) + sinatra (~> 1.4.6) + sinatra-json (~> 0.1.0) + +BUNDLED WITH + 1.10.6 diff --git a/README.md b/README.md new file mode 100644 index 00000000..74aea6a9 --- /dev/null +++ b/README.md @@ -0,0 +1,55 @@ +# CVEServer + +Simple REST-style web service for the CVE searching + +# Requirements + + * Ruby 2.x.x + * Mongo + * Ruby bundler + +# Installation + + * Clone our repository + + $ git clone https://github.com/SpiderLabs/cve_server.git + + * Install the ruby dependencies + + $ bundle install + + * Download the raw data from the National Vulnerability Database + + $ ./bin/nvd_downloader + + * Configure your database + + $ vi config/database.yml + + * Create and populate the database for you environment + + $ RACK_ENV=development ./bin/seed + + * Create and populate the database + + $ RACK_ENV=development ./bin/seed + + * Start the server + + $ RACK_ENV=development puma + +# Using the API + + * Search for an specific CVE using its ID + + http://localhost:port/v1/cve/CVE-ID + + * Search for CVEs related to any cpe + + http://localhost:port/v1/cpe/php:php + + * List all the available CPEs + + http://localhost:port/v1/cpe/ + + http://localhost:port/v1/cpe/microsoft:windows diff --git a/bin/nvd_downloader b/bin/nvd_downloader new file mode 100755 index 00000000..84e75839 --- /dev/null +++ b/bin/nvd_downloader @@ -0,0 +1,72 @@ +#!/usr/bin/env ruby +$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) +require 'nokogiri' +require 'net/http' +require 'cve_server' + +def fetch_page(url) + uri = URI.parse(url) + use_ssl = uri.scheme == 'https' + Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http| + response = http.request_get(uri) + case response + when Net::HTTPSuccess then + response + when Net::HTTPRedirection then + location = response['location'] + fail "redirected to #{location}" + else + response.value + end + end +end + +def download_file(url, dest_path) + uri = URI.parse(url) + use_ssl = uri.scheme == 'https' + Net::HTTP.start(uri.host, uri.port, use_ssl: use_ssl) do |http| + http.request_get(uri.path) do |response| + case response + when Net::HTTPSuccess then + f = File.open(dest_path, 'w') + response.read_body do |seg| + f << seg + sleep 0.005 + end + f.close + when Net::HTTPRedirection then + location = response['location'] + fail "redirected to #{location}" + else + fail "Unable to download #{url}" + end + end + end +end + +def dest_path(link) + filename = link.split('/').last + File.join(CVEServer::Boot.config.raw_data_path, filename) +end + +response = fetch_page('https://nvd.nist.gov/download.cfm') + +if response.is_a?(Net::HTTPSuccess) + @doc = Nokogiri::HTML(response.body) + xml_file_path = '//td[@class="xml-file-type file-20"]' + @doc.xpath('//html').xpath(xml_file_path).each do |td| + link = td.xpath('a').first['href'] + next unless link =~ /.gz$/ + + dest_path = dest_path(link) + downloaded_path = [dest_path, '.download'].join + + puts "Downloading file from #{link}.." + download_file(link, downloaded_path) + + if File.exist?(downloaded_path) && File.size?(downloaded_path) + File.delete(dest_path) if File.exist?(dest_path) + File.rename(downloaded_path, dest_path) + end + end +end diff --git a/bin/seed b/bin/seed new file mode 100755 index 00000000..39aa3f51 --- /dev/null +++ b/bin/seed @@ -0,0 +1,18 @@ +#!/usr/bin/env ruby +$LOAD_PATH.unshift File.expand_path(File.join('..', '..', '/lib'), __FILE__) +require 'zlib' +require 'cve_server' +require 'cve_server/nvd/reader' + +files = File.join(CVEServer::Boot.config.raw_data_path, '*.xml.gz') + +CVEServer::Cve.drop_all +Dir.glob(files).sort.each do |infile| + puts "Uncompressing #{infile}" + input = Zlib::GzipReader.open(infile).read + @doc = CVEServer::NVD::Reader.new(input) + puts 'Exporting data into the CVE collection' + CVEServer::Cve.bulk_create(@doc.all_cve) +end +puts "Reducing the cpe list" +CVEServer::Cve.reduce_cpes diff --git a/config.ru b/config.ru new file mode 100644 index 00000000..c13e10d2 --- /dev/null +++ b/config.ru @@ -0,0 +1,4 @@ +$LOAD_PATH.unshift File.expand_path('../lib', __FILE__) +require 'cve_server/app' + +run CVEServer::App diff --git a/config/database.yml b/config/database.yml new file mode 100644 index 00000000..8c3cce7b --- /dev/null +++ b/config/database.yml @@ -0,0 +1,17 @@ +production: + database: cves_production + adapter: mongo + host: 127.0.0.1 + port: 27017 + +development: + database: cves_development + adapter: mongo + host: 127.0.0.1 + port: 27017 + +test: + database: cves_test + adapter: mongo + host: 127.0.0.1 + port: 27017 diff --git a/config/deploy.rb b/config/deploy.rb new file mode 100644 index 00000000..6b196fb2 --- /dev/null +++ b/config/deploy.rb @@ -0,0 +1,72 @@ +lock '3.4.0' +set :application, 'cve_server' +set :repo_url, 'git@github.com:SpiderLabs/cve_server.git' +set :branch, 'master' # Default branch is :master +set :deploy_to, '/home/deployer/cve_server' +set :stage, :production +set :pty, false +set :linked_dirs, fetch(:linked_dirs, []).push('nvd_data', 'log', 'tmp') + +set :puma_rackup, -> { File.join(current_path, 'config.ru') } +set :puma_state, "#{shared_path}/tmp/pids/puma.state" +set :puma_pid, "#{shared_path}/tmp/pids/puma.pid" +set :puma_bind, "unix://#{shared_path}/tmp/sockets/puma.sock" #accept array for multi-bind +set :puma_default_control_app, "unix://#{shared_path}/tmp/sockets/pumactl.sock" +set :puma_conf, "#{shared_path}/puma.rb" +set :puma_access_log, "#{shared_path}/log/puma_access.log" +set :puma_error_log, "#{shared_path}/log/puma_error.log" +set :puma_role, :app +set :puma_env, fetch(:rack_env, fetch(:rails_env, 'production')) +set :puma_threads, [0, 16] +set :puma_workers, 0 +set :puma_worker_timeout, nil +set :puma_init_active_record, false +set :puma_preload_app, true + +namespace :deploy do + + namespace :symlink do + desc 'Symlink linked directories' + task :linked_dirs do + next unless any? :linked_dirs + on release_roles :all do + execute :mkdir, '-pv', linked_dir_parents(shared_path) + execute :mkdir, '-pv', shared_path.join('tmp/sockets') + execute :mkdir, '-pv', shared_path.join('tmp/pids') + + fetch(:linked_dirs).each do |dir| + target = release_path.join(dir) + source = shared_path.join(dir) + unless test "[ -L #{target} ]" + if Dir.exist?(target) + execute :rm, '-rf', target + end + execute :ln, '-s', source, target + end + end + end + end + end + + desc 'download the nvd reports' + task :download_nvd_reports do + on fetch(:bundle_servers) do + within release_path do + with fetch(:bundle_env_variables, {}) do + execute :bundle, 'exec', './bin/nvd_downloader' + end + end + end + end + + desc 'reload the database with seed data' + task :seed do + on fetch(:bundle_servers) do + within release_path do + with fetch(:bundle_env_variables, {}) do + execute :bundle, 'exec', "./bin/seed RACK_ENV=#{fetch(:rack_env,{})}" + end + end + end + end +end diff --git a/config/deploy/production.rb b/config/deploy/production.rb new file mode 100644 index 00000000..812946be --- /dev/null +++ b/config/deploy/production.rb @@ -0,0 +1,2 @@ +server 'YourIP', user: 'deployer', roles: %w{app} +set :rack_env, 'production' diff --git a/config/puma.rb b/config/puma.rb new file mode 100644 index 00000000..cdb0cd00 --- /dev/null +++ b/config/puma.rb @@ -0,0 +1,25 @@ +$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) +require 'cve_server' + +@config = CVEServer::Boot.config + +directory @config.root +rackup File.join(@config.root, 'config.ru') +environment @config.env +daemonize @config.env == 'production' + +pid_dir = File.join(@config.root, 'tmp', 'pids') +Dir.mkdir pid_dir unless Dir.exists? pid_dir + +pidfile File.join(pid_dir, 'puma.pid') +state_path File.join(pid_dir, 'puma.state') +stdout_redirect File.join(@config.root, 'log', 'puma.stdout.log'), File.join(@config.root, 'log', 'puma.stderr.log') + +quiet + +threads 0, 16 + +socket_dir = File.join(@config.root, 'tmp','sockets') +Dir.mkdir socket_dir unless Dir.exists? socket_dir +bind 'tcp://0.0.0.0:9292' +bind "unix://#{socket_dir}/puma.sock?umask=0111" diff --git a/lib/cve_server.rb b/lib/cve_server.rb new file mode 100644 index 00000000..d4669f67 --- /dev/null +++ b/lib/cve_server.rb @@ -0,0 +1,4 @@ +require 'cve_server/boot' +require 'cve_server/cve' +require 'cve_server/cpe' +require 'cve_server/helper' diff --git a/lib/cve_server/app.rb b/lib/cve_server/app.rb new file mode 100644 index 00000000..c1ac8813 --- /dev/null +++ b/lib/cve_server/app.rb @@ -0,0 +1,54 @@ +require 'sinatra' +require 'sinatra/json' +require 'cve_server' + +module CVEServer + # CVEServer::App is the CVE Micro Service API + class App < Sinatra::Base + include CVEServer::Helper + + before do + content_type 'application/json' + end + + get '/v1/cve/:cve' do |cve| + bad_request unless valid_cve?(cve) + + @cve = CVEServer::Cve.find(cve.upcase) + if @cve + json_resp @cve + else + not_found + end + end + + get '/v1/cpe/:cpe' do |cpe| + bad_request unless valid_cpe?(cpe) + + @cves = CVEServer::Cve.all_cpe_like(cpe.downcase) + if @cves.count > 0 + json_resp @cves + else + not_found + end + end + + get '/v1/cpe' do + json_resp CVEServer::Cpe.all + end + + private + + def json_resp(body) + json body, status: 200 + end + + def not_found + halt 404, json(error: 'not-found') + end + + def bad_request + halt 400, json(error: 'invalid-parameters') + end + end +end diff --git a/lib/cve_server/boot.rb b/lib/cve_server/boot.rb new file mode 100644 index 00000000..9f9e2ae4 --- /dev/null +++ b/lib/cve_server/boot.rb @@ -0,0 +1,20 @@ +require 'cve_server/config' +require 'cve_server/db/connection' + +module CVEServer + module Boot + extend self + + def config + @config ||= CVEServer::Config.new + end + + def connection + @conn ||= CVEServer::DB::Connection.new(config) + end + + def db + connection.db if connection.open? + end + end +end diff --git a/lib/cve_server/config.rb b/lib/cve_server/config.rb new file mode 100644 index 00000000..3b97fd37 --- /dev/null +++ b/lib/cve_server/config.rb @@ -0,0 +1,40 @@ +require 'yaml' +require 'bundler/setup' + +ENV['RACK_ENV'] ||= 'development' +Bundler.require(:default, ENV['RACK_ENV'].to_sym) + +module CVEServer + # CVEServer::Config helps to handle the configuration options + class Config + attr_reader :env, :root, :path, :db_settings + + def initialize + @env = ENV['RACK_ENV'] + @root = File.join(File.dirname(__FILE__), '..', '..') + @path = File.join(root, 'config', 'database.yml') + end + + def db_adapter + db_settings['adapter'] + end + + def db_options + db_settings.select { |k, _v| k != 'adapter' } + end + + def raw_data_path + if env == 'test' + File.join(root, 'spec', 'fixtures', 'nvd_data') + else + File.join(root, 'nvd_data') + end + end + + private + + def db_settings + @db_settings ||= YAML.load(File.read(path))[env] + end + end +end diff --git a/lib/cve_server/cpe.rb b/lib/cve_server/cpe.rb new file mode 100644 index 00000000..93b3f14b --- /dev/null +++ b/lib/cve_server/cpe.rb @@ -0,0 +1,14 @@ +require 'cve_server/db/document' + +module CVEServer + # CVEServer::Cve is a module to abstract the database table/collection + # to store and read the CVEs. + class Cpe + include CVEServer::DB::Document + collection_name :cpes + + def self.all + super.collect {|cpe| cpe['_id']}.sort + end + end +end diff --git a/lib/cve_server/cve.rb b/lib/cve_server/cve.rb new file mode 100644 index 00000000..6402ad4e --- /dev/null +++ b/lib/cve_server/cve.rb @@ -0,0 +1,56 @@ +require 'cve_server/db/document' + +module CVEServer + # CVEServer::Cve is a module to abstract the database table/collection + # to store and read the CVEs. + class Cve + include CVEServer::DB::Document + + collection_name :cves + + def self.find(cve) + remove_id(super(id: cve)) + end + + def self.all_cpe_like(cpe) + all(cpes: /#{cpe}/i).collect do |h| + h['id'] + end.uniq.sort + end + + def self.reduce_cpes + map_reduce(mapper, reducer, map_reducer_opts).count + end + + def self.mapper + %q( + function() { + var application_names = []; + this.cpes.forEach(function(raw_cpe, index) { + var re = /\bcpe:\/\w:?([a-z0-9_\%\~\.\-]+?:[a-z0-9_\%\~\.\-]+)?:*\b/; + var cpe = raw_cpe.match(re)[1]; + if ((application_names.indexOf(cpe) < 0) && (cpe)) + application_names.push(cpe); + }); + for (var i = 0; i < application_names.length; i++) { + emit(application_names[i], {count: 1}); + } + } + ) + end + + def self.reducer + %q( + function(key, values) { + var res = { count: 0 }; + values.forEach(function(v) { res.count += v.count; }); + return res; + } + ) + end + + def self.map_reducer_opts + { out: { replace: 'cpes' }, raw: true } + end + end +end diff --git a/lib/cve_server/db/connection.rb b/lib/cve_server/db/connection.rb new file mode 100644 index 00000000..077c0305 --- /dev/null +++ b/lib/cve_server/db/connection.rb @@ -0,0 +1,39 @@ +module CVEServer + module DB + # CVEServer::DB::Connection implements the functionality for + # the database connection handling + class Connection + attr_reader :adapter, :options + attr_accessor :db + + def initialize(config) + @adapter = config.db_adapter + @options = config.db_options + @db = open + end + + def open? + case adapter + when 'mongo' then !db.nil? + end + end + + private + + def open + case adapter + when 'mongo' then mongo_connection + else abort "#{adapter} is not supported" + end + end + + def mongo_connection + require adapter + host_port = [[options['host'], options['port']].join(':')] + database = { database: options['database'] } + ::Mongo::Logger.logger.level = Logger::WARN + ::Mongo::Client.new(host_port, database) + end + end + end +end diff --git a/lib/cve_server/db/document.rb b/lib/cve_server/db/document.rb new file mode 100644 index 00000000..5d5d0cc6 --- /dev/null +++ b/lib/cve_server/db/document.rb @@ -0,0 +1,22 @@ +require 'cve_server/boot' +module CVEServer + # CVEServer::DB::Document adds the database connection and the specific + # code for each database into a class + module DB + module Document + extend self + + def included(base) + base.extend(CVEServer::Boot) + adapter = CVEServer::Boot.config.db_adapter + require "cve_server/db/#{adapter}" + klass = adapter.capitalize + if CVEServer::DB.const_defined?(klass) + base.extend(CVEServer::DB.const_get(klass)) + else + abort "The #{adapter} is not supported" + end + end + end + end +end diff --git a/lib/cve_server/db/mongo.rb b/lib/cve_server/db/mongo.rb new file mode 100644 index 00000000..58425fdd --- /dev/null +++ b/lib/cve_server/db/mongo.rb @@ -0,0 +1,51 @@ +module CVEServer + module DB + # CVEServer::DB::Mongo contains the code for the mongo queries + module Mongo + extend self + + def collection_name(name) + @collection = name.to_sym + end + + def find(args) + db[@collection].find(args).first + end + + def all(pattern = {}) + db[@collection].find(pattern) + end + + def all_sorted_by(attr) + db[@collection].find.sort(attr.to_sym => ::Mongo::Index::ASCENDING) + end + + def exist?(args) + db[@collection].find(args).count > 0 + end + + def create(args) + db[@collection].insert_one(args) + end + + def bulk_create(data) + inserts = data.reduce([]) do |ops, chunk| + ops << { :insert_one => chunk } + end + db[@collection].bulk_write(inserts, ordered: true) + end + + def drop_all + db[@collection].drop + end + + def map_reduce(mapper, reducer, options = {}) + db[@collection].find.map_reduce(mapper, reducer, options) + end + + def remove_id(record) + record.delete_if { |k, _| k == '_id' } if record.is_a? Hash + end + end + end +end diff --git a/lib/cve_server/helper.rb b/lib/cve_server/helper.rb new file mode 100644 index 00000000..3d89aeb4 --- /dev/null +++ b/lib/cve_server/helper.rb @@ -0,0 +1,14 @@ +module CVEServer + module Helper + module_function + + def valid_cve?(cve) + # https://cve.mitre.org/cve/identifiers/syntaxchange.html + cve.match(/^CVE-\d{4}-\d{1,7}$/i) + end + + def valid_cpe?(cpe) + cpe.match(/^[a-z0-9_\%\~\.\-]+(:[a-z0-9_\%\~\.\-]+){0,1}$/i) + end + end +end diff --git a/lib/cve_server/nvd/cvss.rb b/lib/cve_server/nvd/cvss.rb new file mode 100644 index 00000000..2858d36d --- /dev/null +++ b/lib/cve_server/nvd/cvss.rb @@ -0,0 +1,52 @@ +module CVEServer + module NVD + # CVEServer::NVD::Cvss provides an easy way to calculate a vector + # using the metrics from the NVD reports. + class Cvss + def initialize(cvss) + @cvss = cvss || {} + @cvss = keys_to_sym! + end + + def to_hash + @cvss.merge!(vector: raw_vector) if valid_vector? + @cvss.to_hash + end + + protected + + def keys_to_sym! + @cvss.keys.each_with_object({}) { |k, h| h[k.to_sym] = @cvss[k]; h } + end + + def raw_vector + metrics_map.collect do |k, abbrev| + [abbrev, @cvss[k.to_sym].to_s.slice(0).upcase].join(':') + end.join('/') + end + + def valid_vector? + valid_metrics? && valid_score? + end + + def valid_metrics? + metrics_map.keys.none? { |k| @cvss[k.to_sym].nil? } + end + + def valid_score? + @cvss[:score].to_f.between?(0.0, 10.0) + end + + def metrics_map + { + access_vector: 'AV', + access_complexity: 'AC', + authentication: 'Au', + confidentiality_impact: 'C', + integrity_impact: 'I', + availability_impact: 'A' + } + end + end + end +end diff --git a/lib/cve_server/nvd/entry.rb b/lib/cve_server/nvd/entry.rb new file mode 100644 index 00000000..6e3d64ba --- /dev/null +++ b/lib/cve_server/nvd/entry.rb @@ -0,0 +1,62 @@ +require 'cve_server/nvd/cvss' +module CVEServer + module NVD + # CVEServer::NVD::Entry provides an easy way to parse entries from the + # reports from the National CVEServererability Database. + # + # https://nvd.nist.gov/download.cfm + class Entry + def initialize(entry) + @entry = entry + end + + def to_hash + { + id: @entry['id'], + summary: xpath_content('.//vuln:summary'), + cwe: cwe_id, + published_at: xpath_content('.//vuln:published-datetime'), + updated_at: xpath_content('.//vuln:last-modified-datetime'), + cvss: CVEServer::NVD::Cvss.new(cvss).to_hash, + references: references, + cpes: cpes + } + end + + private + + def xpath_content(path, node = @entry) + node.xpath(path)[0].content + end + + def cwe_id + cwe = @entry.xpath('.//vuln:cwe')[0] + cwe['id'] unless cwe.nil? + end + + def cvss + path = './/cvss:base_metrics/cvss:*' + @entry.search(path).each_with_object({}) do |cvss, h| + h[cvss.name.gsub(/-/, '_').to_sym] = cvss.content + h + end + end + + def references + @entry.xpath('.//vuln:references').collect do |node| + { + type: node['reference_type'], + name: xpath_content('.//vuln:source', node), + href: node.xpath('.//vuln:reference')[0]['href'], + content: xpath_content('.//vuln:reference', node) + } + end + end + + def cpes + path = './/vuln:vulnerable-software-list/vuln:product' + @entry.xpath(path).collect(&:content) + end + end + end +end diff --git a/lib/cve_server/nvd/reader.rb b/lib/cve_server/nvd/reader.rb new file mode 100644 index 00000000..f94f5672 --- /dev/null +++ b/lib/cve_server/nvd/reader.rb @@ -0,0 +1,26 @@ +require 'nokogiri' +require 'cve_server/nvd/entry' + +module CVEServer + module NVD + # CVEServer::NVD::Reader provides an easy way to read the reports + # (version 2.0) from the National CVEServererability Database. + # + # https://nvd.nist.gov/download.cfm + class Reader + def initialize(input) + @doc = Nokogiri::XML(input) + end + + def all_cve + @doc.xpath('//xmlns:entry').collect do |entry| + CVEServer::NVD::Entry.new(entry).to_hash + end + end + + def each_cve(&blk) + all_cve.each(&blk) + end + end + end +end diff --git a/log/.gitkeep b/log/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/nvd_data/.gitkeep b/nvd_data/.gitkeep new file mode 100644 index 00000000..e69de29b diff --git a/spec/cve_server/app_spec.rb b/spec/cve_server/app_spec.rb new file mode 100644 index 00000000..1208a8cc --- /dev/null +++ b/spec/cve_server/app_spec.rb @@ -0,0 +1,168 @@ +require 'spec_helper' +require 'cve_server/app' + +describe CVEServer::App do + def app + @app ||= CVEServer::App + end + + describe 'Specs for /v1/cve/:cve' do + describe 'GET /v1/cve/CVE-2014-0001' do + it 'should be successful' do + get '/v1/cve/CVE-2014-0001' + expect(last_response).to be_ok + end + + it 'should be case insensitive' do + get '/v1/cve/cve-2014-0001' + expect(last_response).to be_ok + end + + it 'should return content-type as json' do + get '/v1/cve/CVE-2014-0001' + expect(response_content_type).to eq 'application/json' + end + + it 'shoud not be emtpy' do + get '/v1/cve/CVE-2014-0001' + expect(last_response).not_to be_empty + end + + it 'should return json with the CVE-2014-0001 ID' do + get '/v1/cve/CVE-2014-0001' + expect(json_response['id']).to eq 'CVE-2014-0001' + end + + it 'should return json with the CVE-2014-0001 ID' do + get '/v1/cve/cVe-2014-0001' + expect(json_response['id']).to eq 'CVE-2014-0001' + end + + it 'should expect status equal to 200' do + get '/v1/cve/CVE-2014-0001' + expect(last_response.status).to eq(200) + end + + it 'should expect status equal to 200 using downcase characters' do + get '/v1/cve/cve-2014-0001' + expect(last_response.status).to eq(200) + end + end + + describe 'GET /v1/cve/CVE-1000-0001' do + it 'should return not found error message' do + get '/v1/cve/CVE-1000-0001' + expect(json_response['error']).to eq 'not-found' + end + + it 'should return status equel to 404' do + get '/v1/cve/CVE-1000-0001' + expect(last_response.status).to eq(404) + end + end + + describe 'GET /v1/cve/bad-requests' do + it 'should return invalid parameters message' do + get '/v1/cve/bad-request' + expect(json_response['error']).to eq 'invalid-parameters' + end + + it 'should return status equal to 400' do + get '/v1/cve/bad-request' + expect(last_response.status).to eq(400) + end + end + end + + describe 'Specs for /v1/cve/:cve' do + describe 'GET /v1/cve/apache:camel' do + it 'should be successful' do + get '/v1/cpe/apache:camel' + expect(last_response).to be_ok + end + + it 'should be case insensitive' do + get '/v1/cpe/apache:CAMEL' + expect(last_response).to be_ok + end + + it 'should return content-type as json' do + get '/v1/cpe/apache:camel' + expect(response_content_type).to eq 'application/json' + end + + it 'shoud not be emtpy' do + get '/v1/cpe/apache:camel' + expect(last_response).not_to be_empty + end + + it 'should return json with a CVE array' do + get '/v1/cpe/apache:camel' + expect(json_response).to eq ['CVE-2014-0002', 'CVE-2014-0003'] + end + + it 'should expect status equal to 200' do + get '/v1/cpe/apache:camel' + expect(last_response.status).to eq(200) + end + + it 'should expect status equal to 200 using upcase characters' do + get '/v1/cpe/apache:CAMEL' + expect(last_response.status).to eq(200) + end + end + + describe 'GET /v1/cpe/oracle:mysql' do + it 'should return not found error message' do + get '/v1/cpe/oracle:mysql' + expect(json_response['error']).to eq 'not-found' + end + + it 'should return status equal to 404' do + get '/v1/cpe/oracle:mysql' + expect(last_response.status).to eq(404) + end + end + + describe 'GET /v1/cpe/bad$requests+' do + it 'should return invalid parameters message' do + get '/v1/cpe/bad$request+' + expect(json_response['error']).to eq 'invalid-parameters' + end + + it 'should return status equel to 400' do + get '/v1/cpe/bad$request+' + expect(last_response.status).to eq(400) + end + end + end + + describe 'Specs for /v1/cve' do + describe 'GET /v1/cve' do + it 'should be successful' do + get '/v1/cpe' + expect(last_response).to be_ok + end + + it 'should return content-type as json' do + get '/v1/cpe' + expect(response_content_type).to eq 'application/json' + end + + it 'should return json with an array including the CPE mysql:mysql, pocoo:jinja2, mariadb:mariadb' do + get '/v1/cpe' + expect(json_response).to include('mysql:mysql', 'pocoo:jinja2', 'mariadb:mariadb') + end + + it 'should return json with an array with 18 CPE strings' do + get '/v1/cpe' + expect(json_response.size).to eq 18 + end + + it 'should expect status equal to 200' do + get '/v1/cpe' + expect(last_response.status).to eq(200) + end + end + end +end diff --git a/spec/cve_server/boot_spec.rb b/spec/cve_server/boot_spec.rb new file mode 100644 index 00000000..824e3e11 --- /dev/null +++ b/spec/cve_server/boot_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' +require 'cve_server/boot' + +describe CVEServer::Boot do + describe 'CVEServer::Boot included in a class' do + before :all do + class BootTestClass + include CVEServer::Boot + end + @class = BootTestClass.new + end + + it 'should have a CveService::Config instance in the config method' do + expect(@class.config).to be_an_instance_of(CVEServer::Config) + end + + it 'should have a CveService::DB::Connection instance in the connection method' do + expect(@class.connection).to be_an_instance_of(CVEServer::DB::Connection) + end + + it 'should not have a nil db objectx' do + expect(@class.db).not_to be nil + end + end +end diff --git a/spec/cve_server/config_spec.rb b/spec/cve_server/config_spec.rb new file mode 100644 index 00000000..87822aef --- /dev/null +++ b/spec/cve_server/config_spec.rb @@ -0,0 +1,24 @@ +require 'spec_helper' +require 'cve_server/config' + +describe CVEServer::Config do + before :all do + @conf = CVEServer::Config.new + end + + it 'should be a test environment' do + expect(@conf.env).to eq 'test' + end + + it 'should have the mongo database adapter' do + expect(@conf.db_adapter).to eq 'mongo' + end + + it 'should have mongo connection options' do + expect(@conf.db_options).to include({'database' => 'cves_test' }) + end + + it 'should not have a raw data path empty' do + expect(@conf.raw_data_path).not_to be_empty + end +end diff --git a/spec/cve_server/db/document_spec.rb b/spec/cve_server/db/document_spec.rb new file mode 100644 index 00000000..cd8296a9 --- /dev/null +++ b/spec/cve_server/db/document_spec.rb @@ -0,0 +1,18 @@ +require 'spec_helper' +require 'cve_server/db/document' + +describe CVEServer::DB::Document do + describe 'CVEServer::DB::Document included in a class' do + before :all do + # Dummy class + class Cve + include CVEServer::DB::Document + collection_name :cves + end + end + + it 'should find one record' do + expect(Cve.find(id: 'CVE-2014-0001')).to have_key('id') + end + end +end diff --git a/spec/cve_server/helper_spec.rb b/spec/cve_server/helper_spec.rb new file mode 100644 index 00000000..7dc11f74 --- /dev/null +++ b/spec/cve_server/helper_spec.rb @@ -0,0 +1,111 @@ +require 'spec_helper' +require 'cve_server/helper' + +module HelperSpec + module_function + + def cves + [ + 'CVE-2011-1529', + 'CVE-2011-1530', + 'CVE-2011-1624', + 'CVE-2011-1625', + 'CVE-2011-1640', + 'CVE-2011-1737', + 'CVE-2011-1738', + 'CVE-2012-0814', + 'cve-2013-2125', + 'cve-2013-4548', + 'CVE-2014-1692', + 'CVE-2014-2532', + 'CVE-2014-2653', + 'CVE-2014-7250', + 'CVE-2014-9278', + 'CVE-2014-9424' + ] + end + + def invalid_cves + [ + 'CVE20111529', + 'CVE_2011_1530', + 'CVE*2011*1624', + 'CVE?2011?1624', + 'CVE-\d{4}-\d{4}', + ] + end + + def cpes + [ + '1two:livre_d_or', + '1und1:1%261_online_storage', + '20_20_applications:20_20_auto_gallery', + 'amarok:amarok', + 'amarok:web_frontend', + 'amateras:amateras_sns', + 'apple:mac_os', + 'apple:mac_os_runtime_for_java', + 'apple:mac_os_x', + 'cisco:nac_manager', + 'cisco:netflow_collection_engine', + 'cisco:network_access_control', + 'cisco:network_admission_control', + 'cisco:network_analysis_module', + 'cisco:network_convergence_system_6000', + 'cisco:network_convergence_system_6008', + 'cisco:network_services_manager', + 'cisco:nexus_1000v', + 'cisco:nexus_2148t_fex_switch', + 'cisco:nexus_2224tp_fex_switch', + 'cisco:nexus_2232pp_fex_switch', + 'cisco:nexus_2232tm_fex_switch', + 'cisco:nexus_2248tp_e_fex_switch', + 'cisco:nexus_2248tp_fex_switch', + ] + end + + def invalid_cpes + [ + '::::::::::::', + '*:*:*', + 'amazon:*', + '*:openbsd', + '\*:linux', + '%\?pple:mac_os_x', + ] + end +end + +describe 'CVEServer::Helper' do + before :all do + @module = CVEServer::Helper + end + + describe 'CVE Validations' do + HelperSpec.cves.each do |cve| + it "should validates '#{cve}'" do + expect(@module.valid_cve?(cve)).not_to be eq(nil) + end + end + + HelperSpec.invalid_cves.each do |cve| + it "should not validates '#{cve}'" do + expect(@module.valid_cve?(cve)).to eq(nil) + end + end + end + + describe 'CPE Validations' do + HelperSpec.cpes.each do |cpe| + it "should validates '#{cpe}'" do + expect(@module.valid_cpe?(cpe)).not_to be eq(nil) + end + end + + HelperSpec.invalid_cpes.each do |cpe| + it "should not validates '#{cpe}'" do + expect(@module.valid_cpe?(cpe)).to eq(nil) + end + end + end +end diff --git a/spec/cve_server/nvd/cvss_spec.rb b/spec/cve_server/nvd/cvss_spec.rb new file mode 100644 index 00000000..b088e962 --- /dev/null +++ b/spec/cve_server/nvd/cvss_spec.rb @@ -0,0 +1,168 @@ +require 'spec_helper' +require 'cve_server/nvd/cvss' + +module CvssSpec + module_function + + def cvss + [ + { + cvss: { + score: '7.5', + access_vector: 'NETWORK', + access_complexity: 'LOW', + authentication: 'NONE', + confidentiality_impact: 'PARTIAL', + integrity_impact: 'PARTIAL', + availability_impact: 'PARTIAL', + source: 'http://nvd.nist.gov', + generated_on_datetime: '2014-02-02T20:10:48.000-05:00' + }, + expected_vector: 'AV:N/AC:L/Au:N/C:P/I:P/A:P' + }, + { + cvss: { + score: '4.3', + access_vector: 'NETWORK', + access_complexity: 'MEDIUM', + authentication: 'NONE', + confidentiality_impact: 'NONE', + integrity_impact: 'NONE', + availability_impact: 'PARTIAL', + source: 'http://nvd.nist.gov', + generated_on_datetime: '2015-06-11T12:19:19.957-04:00', + }, + expected_vector: 'AV:N/AC:M/Au:N/C:N/I:N/A:P' + }, + { + cvss: { + 'score': '4.0', + 'access_vector': 'NETWORK', + 'access_complexity': 'LOW', + 'authentication': 'SINGLE_INSTANCE', + 'confidentiality_impact': 'PARTIAL', + 'integrity_impact': 'NONE', + 'availability_impact': 'NONE', + 'source': 'http://nvd.nist.gov', + 'generated_on_datetime': '2015-02-09T13:50:39.923-05:00' + }, + expected_vector: 'AV:N/AC:L/Au:S/C:P/I:N/A:N' + }, + { + cvss: { + 'score': '4.3', + 'access_vector': 'NETWORK', + 'access_complexity': 'MEDIUM', + 'authentication': 'NONE', + 'confidentiality_impact': 'NONE', + 'integrity_impact': 'PARTIAL', + 'availability_impact': 'NONE', + 'source': 'http://nvd.nist.gov', + 'generated_on_datetime': '2012-01-27T09:55:00.000-05:00', + }, + expected_vector: 'AV:N/AC:M/Au:N/C:N/I:P/A:N' + }, + { + cvss: { + 'score': '7.6', + 'access_vector': 'NETWORK', + 'access_complexity': 'HIGH', + 'authentication': 'NONE', + 'confidentiality_impact': 'COMPLETE', + 'integrity_impact': 'COMPLETE', + 'availability_impact': 'COMPLETE', + 'source': 'http://nvd.nist.gov', + 'generated_on_datetime': '2011-03-03T16:58:00.000-05:00' + }, + expected_vector: 'AV:N/AC:H/Au:N/C:C/I:C/A:C' + }, + ] + end + + def invalid_cvss + [ + { + cvss: { + 'score': '11.3', + 'access_vector': 'NETWORK', + 'access_complexity': 'MEDIUM', + 'confidentiality_impact': 'NONE', + 'integrity_impact': 'PARTIAL', + 'availability_impact': 'NONE', + 'source': 'http://nvd.nist.gov', + 'generated_on_datetime': '2012-01-27T09:55:00.000-05:00', + } + }, + { + cvss: { + 'score': '-7.6', + 'access_vector': 'NETWORK', + 'access_complexity': 'HIGH', + 'authentication': 'NONE', + 'availability_impact': 'COMPLETE', + 'source': 'http://nvd.nist.gov', + 'generated_on_datetime': '2011-03-03T16:58:00.000-05:00' + } + }, + ] + end +end + +describe 'CVEServer::NVD::Cvss' do + before :all do + @klass = CVEServer::NVD::Cvss + end + + CvssSpec.cvss.each do |entry| + context 'Passes valid cvss entry' do + before :each do + @cvss = @klass.new(entry[:cvss]) + end + + it "should have a valid vector" do + expect(@cvss.send('valid_vector?')).to be true + end + + it "should have a valid score" do + expect(@cvss.send('valid_score?')).to be true + end + + it "should have valid metrics" do + expect(@cvss.send('valid_metrics?')).to be true + end + + it "should expect the vector #{entry[:expected_vector]}" do + expect(@cvss.send('raw_vector')).to be == entry[:expected_vector] + end + + it "should expect a hash including the vector" do + h = entry[:cvss].merge!(vector: entry[:expected_vector]) + expect(@cvss.to_hash).to be == h + end + end + end + + CvssSpec.invalid_cvss.each do |entry| + context 'Should not validate invalid cvss entry ' do + before :each do + @cvss = @klass.new(entry[:cvss]) + end + + it "should not have a valid vector" do + expect(@cvss.send('valid_vector?')).to be false + end + + it "should not have a valid score" do + expect(@cvss.send('valid_score?')).to be false + end + + it "should not have valid metrics" do + expect(@cvss.send('valid_metrics?')).to be false + end + + it "should expect the original hash" do + expect(@cvss.to_hash).to be == entry[:cvss] + end + end + end +end diff --git a/spec/cve_server/nvd/entry_spec.rb b/spec/cve_server/nvd/entry_spec.rb new file mode 100644 index 00000000..6c5637c7 --- /dev/null +++ b/spec/cve_server/nvd/entry_spec.rb @@ -0,0 +1,26 @@ +require 'spec_helper' +require 'cve_server/nvd/entry' +require 'zlib' +require 'nokogiri' + +describe CVEServer::NVD::Entry do + before :all do + @infile = File.expand_path('../../../fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz', __FILE__) + @xml = Zlib::GzipReader.open(@infile).read + @doc = Nokogiri::XML(@xml) + @entry = @doc.xpath('//xmlns:entry').first + @nvd_entry = CVEServer::NVD::Entry.new(@entry) + end + + it 'should be an instance of CVEServer::NVD::Entry' do + expect(@nvd_entry).to be_an_instance_of(CVEServer::NVD::Entry) + end + + it 'should have the CVE attributes' do + expect(@nvd_entry.to_hash.keys).to include(:id, :summary, :cwe, :published_at, :updated_at, :cvss, :references, :cpes) + end + + it 'should have the CVE ID CVE-2014-0001' do + expect(@nvd_entry.to_hash[:id]).to eq 'CVE-2014-0001' + end +end diff --git a/spec/cve_server/nvd/reader_spec.rb b/spec/cve_server/nvd/reader_spec.rb new file mode 100644 index 00000000..20292ea8 --- /dev/null +++ b/spec/cve_server/nvd/reader_spec.rb @@ -0,0 +1,25 @@ +require 'spec_helper' +require 'cve_server/nvd/reader' +require 'zlib' + +describe CVEServer::NVD::Reader do + before :all do + @infile = File.expand_path('../../../fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz', __FILE__) + @xml = Zlib::GzipReader.open(@infile).read + @nvd_reader = CVEServer::NVD::Reader.new(@xml) + end + + it 'Should return an array with CVEs' do + expect(@nvd_reader.all_cve).to be_a(Array) + end + + it 'Should return 17 entries for CVEs' do + expect(@nvd_reader.all_cve.size).to eq 17 + end + + it 'Should pass each CVE as a Hash' do + @nvd_reader.each_cve do |cve| + expect(cve).to be_an_instance_of(Hash) + end + end +end diff --git a/spec/fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz b/spec/fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz new file mode 100644 index 00000000..dff2511b Binary files /dev/null and b/spec/fixtures/nvd_data/partial-nvdcve-2.0-2014.xml.gz differ diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 00000000..996718da --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,26 @@ +require 'rspec' +require 'rack/test' +require 'json' +require 'simplecov' + +ENV['RACK_ENV'] = 'test' +$LOAD_PATH.unshift File.expand_path('../lib', __FILE__) + +SimpleCov.start + +module CVEServer + module TestHelper + def response_content_type + last_response.headers['Content-Type'] + end + + def json_response + JSON.parse(last_response.body) + end + end +end + +RSpec.configure do |conf| + conf.include Rack::Test::Methods + conf.include CVEServer::TestHelper +end diff --git a/tmp/.gitkeep b/tmp/.gitkeep new file mode 100644 index 00000000..e69de29b